Replace template registry system with git-backed template sources

source/src/commands/destroy.cpp

@@ -79,6 +79,8 @@ namespace dropshell
                     LocalServiceInfo service_info;
                     service_info = get_service_info(server, service);
                     bool service_valid = SIvalid(service_info);
+                    if (service_valid)
+                        gTemplateManager().pull_for_template(service_info.template_name);
                     if (!service_valid)
                         warning << "No valid service definition found for " << service << std::endl;
 

source/src/commands/help.cpp

@@ -158,7 +158,8 @@ int help_handler(const CommandContext& ctx) {
         show_command("create-template");
         info << std::endl;
         show_command("validate-template");
-        show_command("publish-template");
+        info << std::endl;
+        show_command("pull");
     }
     return 0;
 }

source/src/commands/install.cpp

@@ -90,6 +90,9 @@ namespace dropshell
 
             maketitle("Installing " + service + " (" + service_info.template_name + ") on " + server);
 
+            // Pull latest template from git if applicable
+            gTemplateManager().pull_for_template(service_info.template_name);
+
             // Check if template exists
             template_info tinfo = gTemplateManager().get_template_info(service_info.template_name);
             if (!tinfo.is_set())

source/src/commands/list-templates.cpp

@@ -27,7 +27,7 @@ struct ListTemplatesCommandRegister {
             "list-templates",
             "List all available templates and their sources.",
             R"(
-List all available templates from configured local paths and registries.
+List all available templates from template_paths.json sources.
   list-templates    Show template names and where they come from.
             )"
         });

source/src/commands/publish-template.cpp

@@ -1,601 +0,0 @@
-#include "command_registry.hpp"
-#include "config.hpp"
-#include "templates.hpp"
-#include "utils/utils.hpp"
-#include "utils/directories.hpp"
-#include "utils/hash.hpp"
-#include "shared_commands.hpp"
-
-#include <iostream>
-#include <sstream>
-#include <filesystem>
-#include <fstream>
-#include <ctime>
-#include <iomanip>
-#include <cstdlib>
-#include <algorithm>
-
-namespace dropshell {
-
-void publish_template_autocomplete(const CommandContext& ctx);
-int publish_template_handler(const CommandContext& ctx);
-
-static std::vector<std::string> publish_template_name_list = {"publish-template", "publish"};
-
-// Static registration
-struct PublishTemplateCommandRegister {
-    PublishTemplateCommandRegister() {
-        CommandRegistry::instance().register_command({
-            publish_template_name_list,
-            publish_template_handler,
-            publish_template_autocomplete,
-            false, // hidden
-            true,  // requires_config
-            true,  // requires_install
-            1,     // min_args
-            3,     // max_args (--all + optional registry + directory)
-            "publish-template [--all] [REGISTRY] DIRECTORY",
-            "Publish a template to a template registry.",
-            R"HELP(
-Publishes a template directory to a template registry.
-
-Usage:
-  ds publish-template DIRECTORY
-  ds publish-template REGISTRY_NAME DIRECTORY
-  ds publish-template --all DIRECTORY
-  ds publish-template --all REGISTRY_NAME DIRECTORY
-
-Arguments:
-  --all          Publish all templates in subdirectories of DIRECTORY.
-                 Only publishes subdirectories containing template_info.env
-  REGISTRY_NAME  Optional. Name of the template registry to publish to.
-                 If not specified, uses the first registry with a token.
-  DIRECTORY      Path to the template directory (or parent directory with --all).
-
-The template is validated before publishing. Two tags are created:
-  - YYYYMMDD  (e.g., 20251228)
-  - latest
-
-Authentication:
-  Token is resolved in this order:
-  1. Token configured in dropshell.json for the registry
-  2. SOS_WRITE_TOKEN environment variable (if single registry defined)
-
-Requirements:
-  - Template must pass validation
-  - Valid authentication token
-  - curl must be available
-
-Example:
-  ds publish-template ./my-template
-  ds publish-template main ./my-template
-  ds publish-template --all ./templates-dir
-  SOS_WRITE_TOKEN=xxx ds publish-template ./my-template
-            )HELP"
-        });
-    }
-} publish_template_command_register;
-
-
-void publish_template_autocomplete(const CommandContext& ctx) {
-    if (ctx.args.size() == 0) {
-        rawout << "--all" << std::endl;
-        // List registry names
-        std::vector<tRegistryEntry> registries = gConfig().get_template_registry_urls();
-        for (const auto& reg : registries) {
-            if (!reg.token.empty()) {
-                rawout << reg.name << std::endl;
-            }
-        }
-    }
-}
-
-// Get current date as YYYYMMDD (UTC)
-static std::string get_date_tag() {
-    auto now = std::time(nullptr);
-    auto tm = *std::gmtime(&now);
-    std::ostringstream oss;
-    oss << std::put_time(&tm, "%Y%m%d");
-    return oss.str();
-}
-
-// Create tarball of template
-static bool create_tarball(const std::string& template_dir, const std::string& tarball_path) {
-    std::filesystem::path dir_path(template_dir);
-    std::string template_name = dir_path.filename().string();
-    std::string parent_dir = dir_path.parent_path().string();
-
-    std::string cmd = "tar -czf \"" + tarball_path + "\" -C \"" + parent_dir + "\" \"" + template_name + "\" 2>/dev/null";
-    int ret = system(cmd.c_str());
-
-    return ret == 0 && std::filesystem::exists(tarball_path);
-}
-
-// Run curl and capture output
-static bool run_curl(const std::string& cmd, std::string& output, int& http_code) {
-    // Append -w to get HTTP status code
-    std::string full_cmd = cmd + " -w '\\nHTTP_CODE:%{http_code}' 2>&1";
-
-    FILE* pipe = popen(full_cmd.c_str(), "r");
-    if (!pipe) return false;
-
-    char buffer[512];
-    output.clear();
-    while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
-        output += buffer;
-    }
-
-    int ret = pclose(pipe);
-
-    // Extract HTTP code from output
-    size_t code_pos = output.find("HTTP_CODE:");
-    if (code_pos != std::string::npos) {
-        http_code = std::stoi(output.substr(code_pos + 10));
-        output = output.substr(0, code_pos);
-        // Remove trailing newline
-        while (!output.empty() && (output.back() == '\n' || output.back() == '\r')) {
-            output.pop_back();
-        }
-    } else {
-        http_code = 0;
-    }
-
-    return ret == 0 || http_code == 200;
-}
-
-// Check if file already exists on server by hash
-static bool check_file_exists(const std::string& server_url, const std::string& file_hash) {
-    std::string cmd = "curl -s \"" + server_url + "/exists/" + file_hash + "\"";
-    std::string output;
-    int http_code;
-
-    if (!run_curl(cmd, output, http_code)) {
-        return false;
-    }
-
-    // Response is JSON: {"exists": true} or {"exists": false}
-    return output.find("\"exists\":true") != std::string::npos ||
-           output.find("\"exists\": true") != std::string::npos;
-}
-
-// Get unpacked hash for a template:tag from server metadata
-// Returns empty string if not found or error
-static std::string get_remote_unpacked_hash(const std::string& server_url, const std::string& labeltag) {
-    std::string cmd = "curl -s \"" + server_url + "/meta/" + labeltag + "\"";
-    std::string output;
-    int http_code;
-
-    if (!run_curl(cmd, output, http_code) || http_code != 200) {
-        return "";
-    }
-
-    // Parse unpackedhash from JSON response
-    // Look for "unpackedhash":"<hash>" or "unpackedhash": "<hash>"
-    size_t pos = output.find("\"unpackedhash\"");
-    if (pos == std::string::npos) {
-        return "";
-    }
-
-    // Find the colon and opening quote
-    pos = output.find(':', pos);
-    if (pos == std::string::npos) return "";
-
-    pos = output.find('"', pos);
-    if (pos == std::string::npos) return "";
-
-    pos++; // skip opening quote
-    size_t end = output.find('"', pos);
-    if (end == std::string::npos) return "";
-
-    return output.substr(pos, end - pos);
-}
-
-// Upload a new file to the server
-static bool upload_file(const std::string& server_url, const std::string& token,
-                        const std::string& file_path, const std::vector<std::string>& labeltags,
-                        const std::string& unpacked_hash) {
-    // Build labeltags JSON array
-    std::ostringstream labeltags_json;
-    labeltags_json << "[";
-    for (size_t i = 0; i < labeltags.size(); i++) {
-        if (i > 0) labeltags_json << ",";
-        labeltags_json << "\"" << labeltags[i] << "\"";
-    }
-    labeltags_json << "]";
-
-    // Build metadata JSON
-    std::ostringstream metadata;
-    metadata << "{\"labeltags\":" << labeltags_json.str()
-             << ",\"unpackedhash\":\"" << unpacked_hash << "\""
-             << ",\"description\":\"Published by dropshell\"}";
-
-    // Build curl command for multipart upload
-    std::string cmd = "curl -s -X PUT "
-                      "-H \"Authorization: Bearer " + token + "\" "
-                      "-F \"file=@" + file_path + "\" "
-                      "-F 'metadata=" + metadata.str() + "' "
-                      "\"" + server_url + "/upload\"";
-
-    std::string output;
-    int http_code;
-
-    if (!run_curl(cmd, output, http_code)) {
-        error << "Upload failed: " << output << std::endl;
-        return false;
-    }
-
-    if (http_code != 200) {
-        error << "Upload failed with HTTP " << http_code << ": " << output << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
-// Update metadata for an existing file
-static bool update_metadata(const std::string& server_url, const std::string& token,
-                           const std::string& file_hash, const std::vector<std::string>& labeltags,
-                           const std::string& unpacked_hash) {
-    // Build labeltags JSON array
-    std::ostringstream labeltags_json;
-    labeltags_json << "[";
-    for (size_t i = 0; i < labeltags.size(); i++) {
-        if (i > 0) labeltags_json << ",";
-        labeltags_json << "\"" << labeltags[i] << "\"";
-    }
-    labeltags_json << "]";
-
-    // Build metadata JSON
-    std::ostringstream metadata;
-    metadata << "{\"labeltags\":" << labeltags_json.str()
-             << ",\"unpackedhash\":\"" << unpacked_hash << "\""
-             << ",\"description\":\"Published by dropshell\"}";
-
-    // Build the full request body
-    std::ostringstream body;
-    body << "{\"hash\":\"" << file_hash << "\",\"metadata\":" << metadata.str() << "}";
-
-    // Build curl command for JSON update
-    std::string cmd = "curl -s -X PUT "
-                      "-H \"Authorization: Bearer " + token + "\" "
-                      "-H \"Content-Type: application/json\" "
-                      "-d '" + body.str() + "' "
-                      "\"" + server_url + "/update\"";
-
-    std::string output;
-    int http_code;
-
-    if (!run_curl(cmd, output, http_code)) {
-        error << "Update failed: " << output << std::endl;
-        return false;
-    }
-
-    if (http_code != 200) {
-        error << "Update failed with HTTP " << http_code << ": " << output << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
-// Publish a file to the registry (upload or update metadata if exists)
-static bool publish_to_registry(const std::string& server_url, const std::string& token,
-                                const std::string& file_path, const std::vector<std::string>& labeltags,
-                                const std::string& unpacked_hash) {
-    // Calculate SHA256 hash of the tarball for deduplication
-    std::string file_hash = hash_file(file_path);
-    if (file_hash.empty()) {
-        error << "Failed to calculate file hash" << std::endl;
-        return false;
-    }
-
-    // Check if file already exists on server
-    bool exists = check_file_exists(server_url, file_hash);
-
-    if (exists) {
-        // File exists, just update metadata (adds new labeltags)
-        return update_metadata(server_url, token, file_hash, labeltags, unpacked_hash);
-    } else {
-        // File doesn't exist, upload it
-        return upload_file(server_url, token, file_path, labeltags, unpacked_hash);
-    }
-}
-
-// Check if a directory is a valid template (has template_info.env at root)
-static bool is_valid_template_dir(const std::filesystem::path& dir_path) {
-    return std::filesystem::exists(dir_path / "template_info.env");
-}
-
-// Publish a single template directory
-// Returns: 0 = success, 1 = error, 2 = skipped (not a template), 3 = unchanged (already up to date)
-static int publish_single_template(const std::string& template_dir, const std::string& server_url,
-                                   const std::string& token, bool quiet = false, bool skip_if_unchanged = false) {
-    std::filesystem::path dir_path(template_dir);
-    std::string template_name = dir_path.filename().string();
-
-    if (!quiet) {
-        maketitle("Publishing template: " + template_name);
-        info << "Directory: " << template_dir << std::endl;
-        std::cout << std::endl;
-    }
-
-    // Validate template
-    if (!quiet) info << "=== Validating Template ===" << std::endl;
-    if (!template_manager::test_template(template_dir)) {
-        error << "Template '" << template_name << "' validation failed." << std::endl;
-        if (!quiet) info << "Run: ds validate-template " << template_dir << std::endl;
-        return 1;
-    }
-    if (!quiet) info << "  ✓ Template is valid" << std::endl;
-    if (!quiet) std::cout << std::endl;
-
-    // Calculate directory hash
-    if (!quiet) info << "=== Calculating Hash ===" << std::endl;
-    std::string unpacked_hash = hash_directory_recursive(template_dir);
-    if (unpacked_hash.empty()) {
-        error << "Failed to calculate directory hash for " << template_name << std::endl;
-        return 1;
-    }
-    if (!quiet) info << "  Hash: " << unpacked_hash << std::endl;
-
-    // Check if unchanged (compare with remote :latest)
-    if (skip_if_unchanged) {
-        std::string remote_hash = get_remote_unpacked_hash(server_url, template_name + ":latest");
-        if (!remote_hash.empty() && remote_hash == unpacked_hash) {
-            if (!quiet) {
-                info << "  ✓ Unchanged (matches remote)" << std::endl;
-                std::cout << std::endl;
-            }
-            return 3; // unchanged
-        }
-    }
-    if (!quiet) std::cout << std::endl;
-
-    // Create temp directory and tarball
-    std::string temp_dir = std::filesystem::temp_directory_path().string() + "/dropshell-publish-" + std::to_string(getpid());
-    std::filesystem::create_directories(temp_dir);
-
-    // Cleanup on exit
-    struct TempDirCleaner {
-        std::string path;
-        ~TempDirCleaner() {
-            std::filesystem::remove_all(path);
-        }
-    } cleaner{temp_dir};
-
-    if (!quiet) info << "=== Creating Package ===" << std::endl;
-    std::string tarball_path = temp_dir + "/" + template_name + ".tgz";
-    if (!create_tarball(template_dir, tarball_path)) {
-        error << "Failed to create tarball for " << template_name << std::endl;
-        return 1;
-    }
-
-    auto file_size = std::filesystem::file_size(tarball_path);
-    if (!quiet) info << "  Created " << template_name << ".tgz (" << (file_size / 1024) << " KB)" << std::endl;
-    if (!quiet) std::cout << std::endl;
-
-    // Publish with tags
-    if (!quiet) info << "=== Publishing to Registry ===" << std::endl;
-
-    std::string date_tag = get_date_tag();
-    std::vector<std::string> labeltags = {
-        template_name + ":" + date_tag,
-        template_name + ":latest"
-    };
-
-    if (!quiet) info << "  Tags: " << date_tag << ", latest" << std::endl;
-    if (!quiet) info << "  Uploading..." << std::endl;
-
-    if (!publish_to_registry(server_url, token, tarball_path, labeltags, unpacked_hash)) {
-        error << "Failed to publish template " << template_name << std::endl;
-        return 1;
-    }
-
-    if (!quiet) {
-        info << "  ✓ Published successfully" << std::endl;
-        std::cout << std::endl;
-    }
-
-    return 0;
-}
-
-int publish_template_handler(const CommandContext& ctx) {
-    bool publish_all = false;
-    std::string registry_name;
-    std::string target_dir;
-
-    // Parse arguments - check for --all flag
-    std::vector<std::string> args;
-    for (const auto& arg : ctx.args) {
-        if (arg == "--all" || arg == "-a" || arg == "all") {
-            publish_all = true;
-        } else {
-            args.push_back(arg);
-        }
-    }
-
-    // Parse remaining arguments
-    if (args.size() == 1) {
-        target_dir = args[0];
-    } else if (args.size() == 2) {
-        registry_name = args[0];
-        target_dir = args[1];
-    } else if (args.empty()) {
-        error << "Usage: ds publish-template [--all] [REGISTRY] DIRECTORY" << std::endl;
-        return 1;
-    } else {
-        error << "Too many arguments" << std::endl;
-        error << "Usage: ds publish-template [--all] [REGISTRY] DIRECTORY" << std::endl;
-        return 1;
-    }
-
-    // Resolve directory to absolute path
-    std::filesystem::path dir_path(target_dir);
-    if (dir_path.is_relative()) {
-        dir_path = std::filesystem::current_path() / dir_path;
-    }
-    dir_path = std::filesystem::canonical(dir_path);
-    target_dir = dir_path.string();
-
-    // Check directory exists
-    if (!std::filesystem::exists(dir_path) || !std::filesystem::is_directory(dir_path)) {
-        error << "Directory not found: " << target_dir << std::endl;
-        return 1;
-    }
-
-    // Find registry and token
-    std::vector<tRegistryEntry> registries = gConfig().get_template_registry_urls();
-    tRegistryEntry* selected_registry = nullptr;
-    std::string effective_token;
-
-    if (registry_name.empty()) {
-        // Find first registry with a token
-        for (auto& reg : registries) {
-            if (!reg.token.empty()) {
-                selected_registry = &reg;
-                effective_token = reg.token;
-                break;
-            }
-        }
-
-        // Fallback: if no registry has a token, but there's exactly one registry
-        // and SOS_WRITE_TOKEN is set, use that
-        if (!selected_registry && registries.size() == 1) {
-            const char* env_token = std::getenv("SOS_WRITE_TOKEN");
-            if (env_token && env_token[0] != '\0') {
-                selected_registry = &registries[0];
-                effective_token = env_token;
-                info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl;
-            }
-        }
-
-        if (!selected_registry) {
-            error << "No template registry with a token found" << std::endl;
-            info << "Either:" << std::endl;
-            info << "  - Add a token to a registry in dropshell.json" << std::endl;
-            info << "  - Set SOS_WRITE_TOKEN environment variable (if only one registry defined)" << std::endl;
-            return 1;
-        }
-    } else {
-        // Find registry by name
-        for (auto& reg : registries) {
-            if (reg.name == registry_name) {
-                selected_registry = &reg;
-                break;
-            }
-        }
-        if (!selected_registry) {
-            error << "Registry not found: " << registry_name << std::endl;
-            return 1;
-        }
-
-        // Use token from config, or fall back to SOS_WRITE_TOKEN env var
-        if (!selected_registry->token.empty()) {
-            effective_token = selected_registry->token;
-        } else {
-            const char* env_token = std::getenv("SOS_WRITE_TOKEN");
-            if (env_token && env_token[0] != '\0') {
-                effective_token = env_token;
-                info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl;
-            } else {
-                error << "Registry '" << registry_name << "' does not have a token configured" << std::endl;
-                info << "Set token in dropshell.json or SOS_WRITE_TOKEN environment variable" << std::endl;
-                return 1;
-            }
-        }
-    }
-
-    // Build server URL (ensure https://)
-    std::string server_url = selected_registry->url;
-    if (server_url.substr(0, 8) != "https://" && server_url.substr(0, 7) != "http://") {
-        server_url = "https://" + server_url;
-    }
-    // Remove trailing slash if present
-    if (!server_url.empty() && server_url.back() == '/') {
-        server_url.pop_back();
-    }
-
-    if (publish_all) {
-        // Publish all templates in subdirectories
-        maketitle("Publishing all templates");
-        info << "Parent directory: " << target_dir << std::endl;
-        info << "Registry: " << selected_registry->name << " (" << server_url << ")" << std::endl;
-        std::cout << std::endl;
-
-        // Find all subdirectories that are valid templates
-        std::vector<std::filesystem::path> template_dirs;
-        for (const auto& entry : std::filesystem::directory_iterator(dir_path)) {
-            if (entry.is_directory() && is_valid_template_dir(entry.path())) {
-                template_dirs.push_back(entry.path());
-            }
-        }
-
-        // Sort alphabetically
-        std::sort(template_dirs.begin(), template_dirs.end());
-
-        if (template_dirs.empty()) {
-            error << "No valid templates found in " << target_dir << std::endl;
-            info << "Templates must have template_info.env" << std::endl;
-            return 1;
-        }
-
-        info << "Found " << template_dirs.size() << " template(s) to publish:" << std::endl;
-        for (const auto& tdir : template_dirs) {
-            info << "  - " << tdir.filename().string() << std::endl;
-        }
-        std::cout << std::endl;
-
-        int success_count = 0;
-        int unchanged_count = 0;
-        int fail_count = 0;
-        std::vector<std::string> failed_templates;
-
-        for (const auto& tdir : template_dirs) {
-            int result = publish_single_template(tdir.string(), server_url, effective_token, false, true);
-            if (result == 0) {
-                success_count++;
-            } else if (result == 3) {
-                unchanged_count++;
-            } else {
-                fail_count++;
-                failed_templates.push_back(tdir.filename().string());
-            }
-        }
-
-        // Summary
-        std::cout << std::endl;
-        maketitle("Publish Summary");
-        if (success_count > 0) {
-            info << "Published: " << success_count << " template(s)" << std::endl;
-        }
-        if (unchanged_count > 0) {
-            info << "Unchanged: " << unchanged_count << " template(s)" << std::endl;
-        }
-        if (fail_count > 0) {
-            error << "Failed: " << fail_count << " template(s)" << std::endl;
-            for (const auto& name : failed_templates) {
-                error << "  - " << name << std::endl;
-            }
-            return 1;
-        }
-
-        return 0;
-    } else {
-        // Single template publish
-        info << "Registry: " << selected_registry->name << " (" << server_url << ")" << std::endl;
-
-        int result = publish_single_template(target_dir, server_url, effective_token, false);
-
-        if (result == 0) {
-            std::string template_name = dir_path.filename().string();
-            maketitle("Publish Complete");
-            info << "Template '" << template_name << "' published successfully!" << std::endl;
-            info << "Tags: " << get_date_tag() << ", latest" << std::endl;
-            info << "URL:  " << server_url << "/" << template_name << ":latest" << std::endl;
-        }
-
-        return result;
-    }
-}
-
-} // namespace dropshell

source/src/commands/pull.cpp

@@ -0,0 +1,44 @@
+#include "command_registry.hpp"
+#include "templates.hpp"
+#include "utils/output.hpp"
+
+namespace dropshell {
+
+int pull_handler(const CommandContext& ctx);
+
+static std::vector<std::string> pull_name_list = {"pull"};
+
+struct PullCommandRegister {
+    PullCommandRegister() {
+        CommandRegistry::instance().register_command({
+            pull_name_list,
+            pull_handler,
+            nullptr,
+            false, // hidden
+            true,  // requires_config
+            false, // requires_install
+            0,     // min_args
+            0,     // max_args
+            "pull",
+            "Pull/clone all git-backed template sources.",
+            R"(
+Pull the latest changes for all git-backed template sources
+defined in template_paths.json.
+
+If a source directory doesn't exist and has a git URL, it will be cloned.
+If it exists with a .git directory, it will be pulled.
+Local-only sources (no git URL) are skipped.
+            )"
+        });
+    }
+} pull_command_register;
+
+int pull_handler(const CommandContext& ctx) {
+    if (!gTemplateManager().is_loaded()) {
+        error << "Template manager not loaded" << std::endl;
+        return 1;
+    }
+    return gTemplateManager().pull_all() ? 0 : 1;
+}
+
+} // namespace dropshell

source/src/commands/restoredata.cpp

@@ -105,6 +105,8 @@ namespace dropshell
             return 1;
         }
 
+        // Pull latest template from git if applicable
+        gTemplateManager().pull_for_template(service_info.template_name);
 
         if (!gTemplateManager().template_command_exists(service_info.template_name, "backup") || 
             !gTemplateManager().template_command_exists(service_info.template_name, "restore"))

source/src/config.cpp

@@ -46,8 +46,6 @@ bool config::load_config() { // load json config file.
     // Validate the config format - check for required and supported fields
     std::set<std::string> allowed_fields = {
         "server_definition_paths",
-        "template_local_paths",
-        "template_registries",
         "backups_path",
         "log_level",
         "disabled_servers"
@@ -55,25 +53,16 @@ bool config::load_config() { // load json config file.
 
     std::set<std::string> deprecated_fields = {
         "template_registry_URLs",
-        "template_upload_token"
+        "template_upload_token",
+        "template_registries",
+        "template_local_paths"
     };
 
     // Check for deprecated fields
     for (const auto& field : deprecated_fields) {
         if (mConfig.contains(field)) {
-            error << "Config file contains deprecated field '" << field << "'" << std::endl;
+            warning << "Config file contains deprecated field '" << field << "' - ignoring it." << std::endl;
-            error << "Please update your config file to the new format." << std::endl;
+            mConfig.erase(field);
-            if (field == "template_registry_URLs") {
-                error << "Replace 'template_registry_URLs' with 'template_registries' using the format:" << std::endl;
-                error << "  \"template_registries\": [" << std::endl;
-                error << "    {" << std::endl;
-                error << "      \"name\": \"main\"," << std::endl;
-                error << "      \"url\": \"https://templates.dropshell.app\"," << std::endl;
-                error << "      \"token\": \"\"" << std::endl;
-                error << "    }" << std::endl;
-                error << "  ]" << std::endl;
-            }
-            return false;
         }
     }
 
@@ -93,24 +82,6 @@ bool config::load_config() { // load json config file.
         }
     }
 
-    // Validate template_registries format if present
-    if (mConfig.contains("template_registries")) {
-        if (!mConfig["template_registries"].is_array()) {
-            error << "'template_registries' must be an array" << std::endl;
-            return false;
-        }
-        for (const auto& registry : mConfig["template_registries"]) {
-            if (!registry.is_object()) {
-                error << "Each registry in 'template_registries' must be an object" << std::endl;
-                return false;
-            }
-            if (!registry.contains("name") || !registry.contains("url")) {
-                error << "Each registry must have 'name' and 'url' fields" << std::endl;
-                return false;
-            }
-        }
-    }
-    
     // Validate log_level if present
     if (mConfig.contains("log_level")) {
         if (!mConfig["log_level"].is_string()) {
@@ -131,15 +102,6 @@ bool config::load_config() { // load json config file.
     return true;
 }
 
-void _append(std::vector<std::string> & a, const std::vector<std::string> & b) {
-    if (b.empty())
-        return;
-    if (a.empty())
-        a = b;
-    else
-        a.insert(std::end(a), std::begin(b), std::end(b));
-}
-
 bool config::save_config()
 {
     std::string config_path = localfile::dropshell_json();
@@ -160,16 +122,6 @@ bool config::save_config()
         mConfig["server_definition_paths"] = {
             dropshell_base + "/servers"
         };
-        mConfig["template_local_paths"] = {
-            dropshell_base + "/local_templates"
-        };
-        mConfig["template_registries"] = nlohmann::json::array({
-            nlohmann::json::object({
-                {"name", "main"},
-                {"url", "https://templates.dropshell.app"},
-                {"token", ""}
-            })
-        });
         mConfig["backups_path"] = dropshell_base + "/backups";
         mConfig["log_level"] = "info";  // Default log level
     }
@@ -187,9 +139,7 @@ bool config::save_config()
 
 bool config::create_aux_directories()
 {
-    std::vector<std::string> paths;
+    std::vector<std::string> paths = get_local_server_definition_paths();
-    _append(paths, get_local_template_paths());
-    _append(paths, get_local_server_definition_paths());
     for (auto & p : paths)
         if (!std::filesystem::exists(p))
         {
@@ -209,27 +159,6 @@ bool config::is_agent_installed()
     return std::filesystem::exists(localfile::bb64());
 }
 
-std::vector<tRegistryEntry> config::get_template_registry_urls() {
-    nlohmann::json template_registries = mConfig["template_registries"];
-    std::vector<tRegistryEntry> registries;
-    for (auto &registry : template_registries) {
-        if (registry.is_object() && !registry.empty())
-            registries.push_back(tRegistryEntry(registry));
-    }
-    return registries;
-}
-
-std::vector<std::string> config::get_local_template_paths()
-{
-    nlohmann::json template_local_paths = mConfig["template_local_paths"];
-    std::vector<std::string> paths;
-    for (auto &path : template_local_paths) {
-        if (path.is_string() && !path.empty())
-            paths.push_back(path);
-    }
-    return paths;
-}
-
 std::vector<std::string> config::get_local_server_definition_paths() {
     nlohmann::json server_definition_paths = mConfig["server_definition_paths"];
     std::vector<std::string> paths;
@@ -248,14 +177,6 @@ std::string config::get_server_create_path()
     return paths[0];
 }
 
-std::string config::get_template_create_path()
-{
-    std::vector<std::string> paths = get_local_template_paths();
-    if (paths.empty())
-        return "";
-    return paths[0];
-}
-
 std::string config::get_backups_path()
 {
     nlohmann::json backups_path = mConfig["backups_path"];
@@ -267,34 +188,6 @@ std::string config::get_backups_path()
     return "";
 }
 
-dropshell::tRegistryEntry::tRegistryEntry(nlohmann::json json)
-{
-    valid = false;
-    if (json.is_object() && !json.empty()) {
-        for (auto &[key, value] : json.items()) {
-            if (value.is_string() && !value.empty())
-                switch (switchhash(key.c_str())) {
-                    case switchhash("name"):
-                        name = value;
-                        break;
-                    case switchhash("url"):
-                        url = value;
-                        break;
-                    case switchhash("token"):
-                        token = value;
-                        break;
-                    default:
-                        break;
-                }
-            }
-        valid = (!url.empty()&&!name.empty()); // token can be empty.
-    }
-}
-
-tRegistryEntry::~tRegistryEntry()
-{
-}
-
 std::string config::get_log_level() const
 {
     if (!mIsConfigSet || !mConfig.contains("log_level"))

source/src/config.hpp

@@ -8,19 +8,6 @@
 
 namespace dropshell {
 
-class tRegistryEntry {
-
-    public:
-        tRegistryEntry(nlohmann::json json);
-        ~tRegistryEntry();
-
-    public:
-        std::string name;
-        std::string url;
-        std::string token;
-        bool valid;
-};
-
 class config {
     public:
         config();
@@ -33,12 +20,9 @@ class config {
         bool is_config_set() const;
         static bool is_agent_installed();
 
-        std::vector<tRegistryEntry> get_template_registry_urls();
-        std::vector<std::string> get_local_template_paths();
         std::vector<std::string> get_local_server_definition_paths();
 
         std::string get_server_create_path();
-        std::string get_template_create_path(); 
         std::string get_backups_path();
 
         std::string get_log_level() const;

source/src/services.cpp

@@ -23,7 +23,7 @@ namespace dropshell
                !service_info.user.empty();
     }
 
-    std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name, bool skip_update)
+    std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name)
     {
         std::vector<LocalServiceInfo> services;
 
@@ -49,7 +49,7 @@ namespace dropshell
                         std::string dirname = entry.path().filename().string();
                         if (dirname.empty() || dirname[0] == '.' || dirname[0] == '_')
                             continue;
-                        auto service = get_service_info(server_name, dirname, skip_update);
+                        auto service = get_service_info(server_name, dirname);
                         if (!service.local_service_path.empty())
                             services.push_back(service);
                         else
@@ -72,7 +72,7 @@ namespace dropshell
         return it->second == "true";
     }
 
-    LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool skip_update)
+    LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name)
     {
 
         if (server_name.empty() || service_name.empty())
@@ -114,15 +114,12 @@ namespace dropshell
             }
             service.template_name = it->second;
 
-            tinfo = gTemplateManager().get_template_info(service.template_name,skip_update);
+            tinfo = gTemplateManager().get_template_info(service.template_name);
             if (!tinfo.is_set())
             {
-                // Template not found - this means it's not available locally OR from registry
+                error << "Template '" << service.template_name << "' not found in local template paths" << std::endl;
-                error << "Template '" << service.template_name << "' not found locally or in registry" << std::endl;
                 return LocalServiceInfo();
             }
-            
-            // Template is available (either locally or downloaded from registry)
             service.local_template_path = tinfo.local_template_path();
 
             { // set the user.
@@ -208,70 +205,4 @@ namespace dropshell
         return backups;
     }
 
-    // bool get_all_service_env_vars(const std::string &server_name, const std::string &service_name, ordered_env_vars &all_env_vars)
-    // {
-    //     clear_vars(all_env_vars);
-
-    //     if (localpath::service(server_name, service_name).empty() || !fs::exists(localpath::service(server_name, service_name)))
-    //     {
-    //         error << "Service not found: " << service_name << " on server " << server_name << std::endl;
-    //         return false;
-    //     }
-
-    //     // Lambda function to load environment variables from a file
-    //     auto load_env_file = [&all_env_vars](const std::string &file)
-    //     {
-    //         if (!file.empty() && std::filesystem::exists(file))
-    //         {
-    //             ordered_env_vars env_vars;
-    //             envmanager env_manager(file);
-    //             env_manager.load();
-    //             env_manager.get_all_variables(env_vars);
-    //             merge_vars(all_env_vars, env_vars);
-    //         }
-    //         else
-    //             warning << "Expected environment file not found: " << file << std::endl;
-    //     };
-
-
-    //     // add in some simple variables first, as others below may depend on/use these in bash.
-    //     set_var(all_env_vars, "SERVER", server_name);
-    //     set_var(all_env_vars, "SERVICE", service_name);
-    //     set_var(all_env_vars, "DOCKER_CLI_HINTS", "false"); // turn off docker junk.
-
-    //     // Load environment files
-    //     load_env_file(localfile::service_env(server_name, service_name));
-
-    //     std::string template_name = get_var(all_env_vars, "TEMPLATE");
-    //     if (template_name.empty())
-    //     {
-    //         error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
-    //         return false;
-    //     }
-    //     auto tinfo = gTemplateManager().get_template_info(template_name, true); // skip updates.
-    //     if (!tinfo.is_set())
-    //     {
-    //         // Template is not available locally or from registry
-    //         error << "Template '" << template_name << "' not found locally or in registry" << std::endl;
-    //         return false;
-    //     }
-    //     ASSERT(std::filesystem::exists(tinfo.local_template_info_env_path()));
-    //     load_env_file(tinfo.local_template_info_env_path());
-
-    //     std::string user = get_var(all_env_vars, "SSH_USER");
-    //     if (user.empty())
-    //     {
-    //         error << "SSH_USER variable not defined in service " << service_name << " on server " << server_name << std::endl;
-    //         info << "This variable definition is always required, and usually set in the "<<filenames::service_env << " file." << std::endl;
-    //         info << "Please check " << localfile::service_env(server_name, service_name) << std::endl;
-    //         return false;
-    //     }
-
-    //     // more additional, these depend on others above.
-    //     set_var(all_env_vars, "CONFIG_PATH", remotepath(server_name, user).service_config(service_name));
-    //     set_var(all_env_vars, "AGENT_PATH", remotepath(server_name, user).agent());
-
-    //     return true;
-    // }
-
 } // namespace dropshell

source/src/services.hpp

@@ -4,7 +4,6 @@
 #include <string>
 #include <vector>
 #include <set>
-//#include <map>
 #include "utils/ordered_env.hpp"
 
 namespace dropshell {
@@ -18,15 +17,13 @@ namespace dropshell {
         bool requires_host_root;
         bool requires_docker;
         bool requires_docker_root;
-        //bool service_template_hash_match;
     };
 
     bool SIvalid(const LocalServiceInfo& service_info);
 
-    // if skip_update, don't check for updates to the service template.
+    std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name);
-    std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name, bool skip_update=false);
 
-    LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool skip_update=false);
+    LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name);
     std::set<std::string> get_used_commands(const std::string& server_name, const std::string& service_name);
 
     // list all backups for a given service (across all servers)

source/src/templates.hpp

@@ -1,26 +1,31 @@
+#pragma once
+
 #include <string>
 #include <vector>
 #include <filesystem>
-#include <memory>
+#include <map>
 #include <set>
 
 #include "config.hpp"
-#define JSON_INLINE_ALL
-#include <nlohmann/json.hpp>
-
 
 namespace dropshell {
 
-typedef enum template_source_type {
+// Represents one entry from template_paths.json
-    TEMPLATE_SOURCE_TYPE_LOCAL,
+struct TemplateSource {
-    TEMPLATE_SOURCE_TYPE_REGISTRY,
+    std::string local_path;    // Git checkout root or local directory
-    TEMPLATE_SOURCE_NOT_SET
+    std::string git_url;       // Optional, empty if local-only
-} template_source_type;
+};
+
+// Represents one resolved template
+struct ResolvedTemplate {
+    std::filesystem::path template_dir;
+    size_t source_index;       // Index into mSources
+};
 
 class template_info {
     public:
         template_info() : mIsSet(false) {}
-        template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool skip_update);
+        template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path);
         virtual ~template_info() {}
         bool is_set() const { return mIsSet; }
         std::string name() const { return mTemplateName; }
@@ -33,55 +38,11 @@ class template_info {
     private:
         std::string mTemplateName;
         std::string mLocationID;
-        std::filesystem::path mTemplateLocalPath; // source or cache.
+        std::filesystem::path mTemplateLocalPath;
         bool mTemplateValid;
         bool mIsSet;
 };
 
-class template_source_interface {
-    public:
-        virtual ~template_source_interface() {}
-        virtual std::set<std::string> get_template_list() = 0;
-        virtual bool has_template(const std::string& template_name) = 0;
-        virtual template_info get_template_info(const std::string& template_name, bool skip_update=false) = 0;
-        virtual bool template_command_exists(const std::string& template_name,const std::string& command) = 0;
-
-        virtual std::string get_description() = 0;
-};
-
-class template_source_registry : public template_source_interface {
-    public:
-        template_source_registry(tRegistryEntry registry) : mRegistry(registry) {}
-
-        ~template_source_registry() {}
-
-        std::set<std::string> get_template_list();
-        bool has_template(const std::string& template_name);
-        template_info get_template_info(const std::string& template_name, bool skip_update=false);
-        bool template_command_exists(const std::string& template_name,const std::string& command);
-
-        std::string get_description() { return "Registry: " + mRegistry.name + " (" + mRegistry.url + ")"; }
-    private:
-        std::filesystem::path get_cache_dir();
-    private:
-        tRegistryEntry mRegistry;
-        std::vector<nlohmann::json> mTemplates; // cached list.
-};
-
-class template_source_local : public template_source_interface {
-    public:
-        template_source_local(std::string local_path) : mLocalPath(local_path) {}
-        ~template_source_local() {}
-        std::set<std::string> get_template_list();
-        bool has_template(const std::string& template_name);
-        template_info get_template_info(const std::string& template_name, bool skip_update=false);
-        bool template_command_exists(const std::string& template_name,const std::string& command);
-
-        std::string get_description() { return "Local: " + mLocalPath.string(); }
-    private:
-        std::filesystem::path mLocalPath;
-};
-
 class template_manager {
     public:
         template_manager() : mLoaded(false) {}
@@ -89,9 +50,9 @@ class template_manager {
 
         std::set<std::string> get_template_list() const;
         bool has_template(const std::string& template_name) const;
-        template_info get_template_info(const std::string& template_name, bool skip_update=false) const; // skip_update = don't check for updates.
+        template_info get_template_info(const std::string& template_name) const;
 
-        bool template_command_exists(const std::string& template_name,const std::string& command) const;
+        bool template_command_exists(const std::string& template_name, const std::string& command) const;
         bool create_template(const std::string& template_name) const;
         static bool test_template(const std::string& template_path);
         static bool check_template_shell_scripts_syntax(const std::string& template_path);
@@ -105,16 +66,26 @@ class template_manager {
         bool is_loaded() const { return mLoaded; }
         int get_source_count() const { return mSources.size(); }
 
+        // Git pull operations
+        bool pull_all();
+        bool pull_for_template(const std::string& template_name);
+
     private:
         static bool required_file(std::string path, std::string template_name);
-        template_source_interface* get_source(const std::string& template_name) const;
+        void resolve_templates();
+        void resolve_source(size_t source_index);
+        std::vector<std::string> parse_list_file(const std::string& local_path) const;
+        std::vector<std::string> discover_templates(const std::string& local_path) const;
+        void write_list_file(const std::string& local_path, const std::vector<std::string>& template_dirs) const;
+        bool git_pull_source(const TemplateSource& source);
+        bool git_clone_source(const TemplateSource& source);
 
     private:
         bool mLoaded;
-        mutable std::vector<std::unique_ptr<template_source_interface>> mSources;
+        std::vector<TemplateSource> mSources;
+        std::map<std::string, ResolvedTemplate> mTemplateMap;
 };
 
 template_manager & gTemplateManager();
 
-
 } // namespace dropshell

source/src/utils/directories.cpp

@@ -128,11 +128,6 @@ namespace dropshell
             return dropshell_dir() + "/temp_files";
         }
 
-        std::string template_cache()
-        {
-            return dropshell_dir() + "/template_cache";
-        }
-
         std::string template_example()
         {
             return agent_local() + "/template_example";
@@ -145,7 +140,6 @@ namespace dropshell
                 dropshell_dir(),
                 agent_local(),
                 agent_remote(),
-                template_cache(),
                 backups(),
                 temp_files()};
             for (auto &p : gConfig().get_local_server_definition_paths())

source/src/utils/directories.hpp

@@ -20,14 +20,6 @@ namespace dropshell {
     //    |-- agent-remote
     //        |-- (remote agent files)
     //    |-- temp_files
-    //    |-- template_cache
-    //        |   |-- <template_name>.json
-    //        |   |-- <template_name>
-    //        |       |-- (...script files...)
-    //        |       |-- template_info.env
-    //        |       |-- config
-    //        |           |-- service.env
-    //        |           |-- (...other service config files...)
 
     // backups_path
     //    |-- katie-_-squashkiwi-_-squashkiwi-test-_-2025-04-28_21-23-59.tgz
@@ -49,6 +41,8 @@ namespace dropshell {
         static const std::string server_json = "server.json";
         static const std::string dropshell_json = "dropshell.json";
         static const std::string ds_run = "ds_run.sh";
+        static const std::string template_paths_json = "template_paths.json";
+        static const std::string dropshell_templates_list = "dropshell-templates.list";
     } // namespace filenames.
 
     namespace localfile {
@@ -77,7 +71,6 @@ namespace dropshell {
 
         std::string backups();
         std::string temp_files();
-        std::string template_cache();
 
         bool create_directories();
     } // namespace local