diff --git a/TEMPLATES.md b/TEMPLATES.md index 72082d4..72705d5 100644 --- a/TEMPLATES.md +++ b/TEMPLATES.md @@ -671,10 +671,10 @@ echo "Installation of ${CONTAINER_NAME} complete" Dropshell includes a comprehensive template validation command that checks for common issues: ```bash -dropshell validate +dropshell validate-template ``` -Aliases: `ds lint`, `ds check-template` +Aliases: `ds validate`, `ds lint` ### What Validate Checks @@ -694,7 +694,7 @@ Aliases: `ds lint`, `ds check-template` ### Example Output ``` -$ ds validate my-template +$ ds validate-template my-template === Structure Validation === ✓ Template structure is valid @@ -718,7 +718,7 @@ Validation Summary: Template 'my-template' has 2 warning(s) The `install` command automatically runs basic syntax checking (`bash -n`) on all template scripts before deploying. If any script has a syntax error, the install fails with a clear message: ``` -[ERR] Template shell scripts have syntax errors. Run 'ds validate my-template' for details. +[ERR] Template shell scripts have syntax errors. Run 'ds validate-template my-template' for details. ``` ### Suppressing Shellcheck Warnings diff --git a/source/src/commands/help.cpp b/source/src/commands/help.cpp index 4702034..c4fa17d 100644 --- a/source/src/commands/help.cpp +++ b/source/src/commands/help.cpp @@ -155,7 +155,7 @@ int help_handler(const CommandContext& ctx) { show_command("create-service"); show_command("create-template"); info << std::endl; - show_command("validate"); + show_command("validate-template"); show_command("publish-template"); } return 0; diff --git a/source/src/commands/publish-template.cpp b/source/src/commands/publish-template.cpp index 57435a0..f8f5164 100644 --- a/source/src/commands/publish-template.cpp +++ b/source/src/commands/publish-template.cpp @@ -36,7 +36,7 @@ struct PublishTemplateCommandRegister { "publish-template [REGISTRY] DIRECTORY", "Publish a template to a template registry.", R"HELP( -Publishes a template directory to a template registry using sos. +Publishes a template directory to a template registry. Usage: ds publish-template DIRECTORY @@ -51,14 +51,20 @@ The template is validated before publishing. Two tags are created: - YYYYMMDD (e.g., 20251228) - latest +Authentication: + Token is resolved in this order: + 1. Token configured in dropshell.json for the registry + 2. SOS_WRITE_TOKEN environment variable (if single registry defined) + Requirements: - Template must pass validation - - Registry must have a token configured in dropshell.json - - curl must be available (for sos download) + - Valid authentication token + - curl must be available Example: ds publish-template ./my-template ds publish-template main ./my-template + SOS_WRITE_TOKEN=xxx ds publish-template ./my-template )HELP" }); } @@ -77,7 +83,7 @@ void publish_template_autocomplete(const CommandContext& ctx) { } } -// Get current date as YYYYMMDD +// Get current date as YYYYMMDD (UTC) static std::string get_date_tag() { auto now = std::time(nullptr); auto tm = *std::gmtime(&now); @@ -86,48 +92,6 @@ static std::string get_date_tag() { return oss.str(); } -// Get architecture string -static std::string get_arch() { - FILE* pipe = popen("uname -m", "r"); - if (!pipe) return "x86_64"; - - char buffer[128]; - std::string result; - if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - result = buffer; - // Remove trailing newline - if (!result.empty() && result.back() == '\n') { - result.pop_back(); - } - } - pclose(pipe); - - if (result == "aarch64" || result == "arm64") { - return "aarch64"; - } - return "x86_64"; -} - -// Download a tool to temp directory -static bool download_tool(const std::string& tool_name, const std::string& temp_dir, std::string& tool_path) { - std::string arch = get_arch(); - tool_path = temp_dir + "/" + tool_name; - - std::string url = "https://getbin.xyz/" + tool_name + ":latest-" + arch; - std::string cmd = "curl -fsSL -o \"" + tool_path + "\" \"" + url + "\" 2>/dev/null"; - - int ret = system(cmd.c_str()); - if (ret != 0) { - return false; - } - - // Make executable - std::string chmod_cmd = "chmod +x \"" + tool_path + "\""; - system(chmod_cmd.c_str()); - - return std::filesystem::exists(tool_path); -} - // Create tarball of template static bool create_tarball(const std::string& template_dir, const std::string& tarball_path) { std::filesystem::path dir_path(template_dir); @@ -140,34 +104,164 @@ static bool create_tarball(const std::string& template_dir, const std::string& t return ret == 0 && std::filesystem::exists(tarball_path); } -// Upload using sos -static bool upload_with_sos(const std::string& sos_path, const std::string& server, - const std::string& tarball, const std::string& label_tag, - const std::string& hash, const std::string& token) { - // Set environment variable for token - setenv("SOS_WRITE_TOKEN", token.c_str(), 1); +// Run curl and capture output +static bool run_curl(const std::string& cmd, std::string& output, int& http_code) { + // Append -w to get HTTP status code + std::string full_cmd = cmd + " -w '\\nHTTP_CODE:%{http_code}' 2>&1"; - std::string cmd = "\"" + sos_path + "\" upload " + server + " \"" + tarball + "\" \"" + label_tag + "\" --metadata \"unpackedhash=" + hash + "\" 2>&1"; - - FILE* pipe = popen(cmd.c_str(), "r"); + FILE* pipe = popen(full_cmd.c_str(), "r"); if (!pipe) return false; char buffer[512]; - std::string output; + output.clear(); while (fgets(buffer, sizeof(buffer), pipe) != nullptr) { output += buffer; } int ret = pclose(pipe); - if (ret != 0) { - error << output << std::endl; + // Extract HTTP code from output + size_t code_pos = output.find("HTTP_CODE:"); + if (code_pos != std::string::npos) { + http_code = std::stoi(output.substr(code_pos + 10)); + output = output.substr(0, code_pos); + // Remove trailing newline + while (!output.empty() && (output.back() == '\n' || output.back() == '\r')) { + output.pop_back(); + } + } else { + http_code = 0; + } + + return ret == 0 || http_code == 200; +} + +// Check if file already exists on server by hash +static bool check_file_exists(const std::string& server_url, const std::string& file_hash) { + std::string cmd = "curl -s \"" + server_url + "/exists/" + file_hash + "\""; + std::string output; + int http_code; + + if (!run_curl(cmd, output, http_code)) { + return false; + } + + // Response is JSON: {"exists": true} or {"exists": false} + return output.find("\"exists\":true") != std::string::npos || + output.find("\"exists\": true") != std::string::npos; +} + +// Upload a new file to the server +static bool upload_file(const std::string& server_url, const std::string& token, + const std::string& file_path, const std::vector& labeltags, + const std::string& unpacked_hash) { + // Build labeltags JSON array + std::ostringstream labeltags_json; + labeltags_json << "["; + for (size_t i = 0; i < labeltags.size(); i++) { + if (i > 0) labeltags_json << ","; + labeltags_json << "\"" << labeltags[i] << "\""; + } + labeltags_json << "]"; + + // Build metadata JSON + std::ostringstream metadata; + metadata << "{\"labeltags\":" << labeltags_json.str() + << ",\"unpackedhash\":\"" << unpacked_hash << "\"" + << ",\"description\":\"Published by dropshell\"}"; + + // Build curl command for multipart upload + std::string cmd = "curl -s -X PUT " + "-H \"Authorization: Bearer " + token + "\" " + "-F \"file=@" + file_path + "\" " + "-F 'metadata=" + metadata.str() + "' " + "\"" + server_url + "/upload\""; + + std::string output; + int http_code; + + if (!run_curl(cmd, output, http_code)) { + error << "Upload failed: " << output << std::endl; + return false; + } + + if (http_code != 200) { + error << "Upload failed with HTTP " << http_code << ": " << output << std::endl; return false; } return true; } +// Update metadata for an existing file +static bool update_metadata(const std::string& server_url, const std::string& token, + const std::string& file_hash, const std::vector& labeltags, + const std::string& unpacked_hash) { + // Build labeltags JSON array + std::ostringstream labeltags_json; + labeltags_json << "["; + for (size_t i = 0; i < labeltags.size(); i++) { + if (i > 0) labeltags_json << ","; + labeltags_json << "\"" << labeltags[i] << "\""; + } + labeltags_json << "]"; + + // Build metadata JSON + std::ostringstream metadata; + metadata << "{\"labeltags\":" << labeltags_json.str() + << ",\"unpackedhash\":\"" << unpacked_hash << "\"" + << ",\"description\":\"Published by dropshell\"}"; + + // Build the full request body + std::ostringstream body; + body << "{\"hash\":\"" << file_hash << "\",\"metadata\":" << metadata.str() << "}"; + + // Build curl command for JSON update + std::string cmd = "curl -s -X PUT " + "-H \"Authorization: Bearer " + token + "\" " + "-H \"Content-Type: application/json\" " + "-d '" + body.str() + "' " + "\"" + server_url + "/update\""; + + std::string output; + int http_code; + + if (!run_curl(cmd, output, http_code)) { + error << "Update failed: " << output << std::endl; + return false; + } + + if (http_code != 200) { + error << "Update failed with HTTP " << http_code << ": " << output << std::endl; + return false; + } + + return true; +} + +// Publish a file to the registry (upload or update metadata if exists) +static bool publish_to_registry(const std::string& server_url, const std::string& token, + const std::string& file_path, const std::vector& labeltags, + const std::string& unpacked_hash) { + // Calculate SHA256 hash of the tarball for deduplication + std::string file_hash = hash_file(file_path); + if (file_hash.empty()) { + error << "Failed to calculate file hash" << std::endl; + return false; + } + + // Check if file already exists on server + bool exists = check_file_exists(server_url, file_hash); + + if (exists) { + // File exists, just update metadata (adds new labeltags) + return update_metadata(server_url, token, file_hash, labeltags, unpacked_hash); + } else { + // File doesn't exist, upload it + return upload_file(server_url, token, file_path, labeltags, unpacked_hash); + } +} + int publish_template_handler(const CommandContext& ctx) { std::string registry_name; std::string template_dir; @@ -201,21 +295,37 @@ int publish_template_handler(const CommandContext& ctx) { std::string template_name = dir_path.filename().string(); - // Find registry + // Find registry and token std::vector registries = gConfig().get_template_registry_urls(); tRegistryEntry* selected_registry = nullptr; + std::string effective_token; // Token to use (may come from config or env var) if (registry_name.empty()) { // Find first registry with a token for (auto& reg : registries) { if (!reg.token.empty()) { selected_registry = ® + effective_token = reg.token; break; } } + + // Fallback: if no registry has a token, but there's exactly one registry + // and SOS_WRITE_TOKEN is set, use that + if (!selected_registry && registries.size() == 1) { + const char* env_token = std::getenv("SOS_WRITE_TOKEN"); + if (env_token && env_token[0] != '\0') { + selected_registry = ®istries[0]; + effective_token = env_token; + info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl; + } + } + if (!selected_registry) { - error << "No template registry with a token found in config" << std::endl; - info << "Add a token to a registry in dropshell.json or specify a registry name" << std::endl; + error << "No template registry with a token found" << std::endl; + info << "Either:" << std::endl; + info << " - Add a token to a registry in dropshell.json" << std::endl; + info << " - Set SOS_WRITE_TOKEN environment variable (if only one registry defined)" << std::endl; return 1; } } else { @@ -230,40 +340,59 @@ int publish_template_handler(const CommandContext& ctx) { error << "Registry not found: " << registry_name << std::endl; return 1; } - if (selected_registry->token.empty()) { - error << "Registry '" << registry_name << "' does not have a token configured" << std::endl; - return 1; + + // Use token from config, or fall back to SOS_WRITE_TOKEN env var + if (!selected_registry->token.empty()) { + effective_token = selected_registry->token; + } else { + const char* env_token = std::getenv("SOS_WRITE_TOKEN"); + if (env_token && env_token[0] != '\0') { + effective_token = env_token; + info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl; + } else { + error << "Registry '" << registry_name << "' does not have a token configured" << std::endl; + info << "Set token in dropshell.json or SOS_WRITE_TOKEN environment variable" << std::endl; + return 1; + } } } - // Extract server from URL (remove https:// prefix) - std::string server = selected_registry->url; - if (server.substr(0, 8) == "https://") { - server = server.substr(8); - } else if (server.substr(0, 7) == "http://") { - server = server.substr(7); + // Build server URL (ensure https://) + std::string server_url = selected_registry->url; + if (server_url.substr(0, 8) != "https://" && server_url.substr(0, 7) != "http://") { + server_url = "https://" + server_url; } // Remove trailing slash if present - if (!server.empty() && server.back() == '/') { - server.pop_back(); + if (!server_url.empty() && server_url.back() == '/') { + server_url.pop_back(); } maketitle("Publishing template: " + template_name); info << "Directory: " << template_dir << std::endl; - info << "Registry: " << selected_registry->name << " (" << server << ")" << std::endl; + info << "Registry: " << selected_registry->name << " (" << server_url << ")" << std::endl; std::cout << std::endl; // Step 1: Validate template info << "=== Validating Template ===" << std::endl; if (!template_manager::test_template(template_dir)) { error << "Template validation failed. Please fix issues before publishing." << std::endl; - info << "Run: ds validate " << template_dir << std::endl; + info << "Run: ds validate-template " << template_dir << std::endl; return 1; } info << " ✓ Template is valid" << std::endl; std::cout << std::endl; - // Step 2: Create temp directory + // Step 2: Calculate directory hash (for unpacked content verification) + info << "=== Calculating Hash ===" << std::endl; + std::string unpacked_hash = hash_directory_recursive(template_dir); + if (unpacked_hash.empty()) { + error << "Failed to calculate directory hash" << std::endl; + return 1; + } + info << " Hash: " << unpacked_hash << std::endl; + std::cout << std::endl; + + // Step 3: Create temp directory and tarball std::string temp_dir = std::filesystem::temp_directory_path().string() + "/dropshell-publish-" + std::to_string(getpid()); std::filesystem::create_directories(temp_dir); @@ -275,30 +404,6 @@ int publish_template_handler(const CommandContext& ctx) { } } cleaner{temp_dir}; - // Step 3: Calculate hash (using built-in hash function) - info << "=== Calculating Hash ===" << std::endl; - std::string hash = hash_directory_recursive(template_dir); - if (hash.empty()) { - error << "Failed to calculate directory hash" << std::endl; - return 1; - } - info << " Hash: " << hash << std::endl; - std::cout << std::endl; - - // Step 4: Download sos tool - info << "=== Downloading Tools ===" << std::endl; - - std::string sos_path; - - info << " Downloading sos..." << std::endl; - if (!download_tool("sos", temp_dir, sos_path)) { - error << "Failed to download sos" << std::endl; - return 1; - } - info << " ✓ sos downloaded" << std::endl; - std::cout << std::endl; - - // Step 5: Create tarball info << "=== Creating Package ===" << std::endl; std::string tarball_path = temp_dir + "/" + template_name + ".tgz"; if (!create_tarball(template_dir, tarball_path)) { @@ -310,35 +415,31 @@ int publish_template_handler(const CommandContext& ctx) { info << " Created " << template_name << ".tgz (" << (file_size / 1024) << " KB)" << std::endl; std::cout << std::endl; - // Step 6: Upload with tags - info << "=== Uploading to Registry ===" << std::endl; + // Step 4: Publish with tags + info << "=== Publishing to Registry ===" << std::endl; std::string date_tag = get_date_tag(); + std::vector labeltags = { + template_name + ":" + date_tag, + template_name + ":latest" + }; - // Upload with date tag - std::string label_date = template_name + ":" + date_tag; - info << " Uploading " << label_date << "..." << std::endl; - if (!upload_with_sos(sos_path, server, tarball_path, label_date, hash, selected_registry->token)) { - error << "Failed to upload " << label_date << std::endl; + info << " Tags: " << date_tag << ", latest" << std::endl; + info << " Uploading..." << std::endl; + + if (!publish_to_registry(server_url, effective_token, tarball_path, labeltags, unpacked_hash)) { + error << "Failed to publish template" << std::endl; return 1; } - info << " ✓ Uploaded " << label_date << std::endl; - // Upload with latest tag - std::string label_latest = template_name + ":latest"; - info << " Uploading " << label_latest << "..." << std::endl; - if (!upload_with_sos(sos_path, server, tarball_path, label_latest, hash, selected_registry->token)) { - error << "Failed to upload " << label_latest << std::endl; - return 1; - } - info << " ✓ Uploaded " << label_latest << std::endl; + info << " ✓ Published successfully" << std::endl; std::cout << std::endl; // Summary maketitle("Publish Complete"); info << "Template '" << template_name << "' published successfully!" << std::endl; info << "Tags: " << date_tag << ", latest" << std::endl; - info << "URL: https://" << server << "/" << template_name << ":latest" << std::endl; + info << "URL: " << server_url << "/" << template_name << ":latest" << std::endl; return 0; } diff --git a/source/src/commands/validate.cpp b/source/src/commands/validate.cpp index ec193e6..08d297a 100644 --- a/source/src/commands/validate.cpp +++ b/source/src/commands/validate.cpp @@ -18,7 +18,7 @@ namespace dropshell { void validate_autocomplete(const CommandContext& ctx); int validate_handler(const CommandContext& ctx); -static std::vector validate_name_list = {"validate", "lint", "check-template"}; +static std::vector validate_name_list = {"validate-template", "validate", "lint"}; // Static registration struct ValidateCommandRegister { @@ -32,7 +32,7 @@ struct ValidateCommandRegister { true, // requires_install 1, // min_args (template name required) 1, // max_args - "validate TEMPLATE", + "validate-template TEMPLATE", "Validate a template's structure, syntax, and common issues.", R"( Validates a dropshell template by checking: