public/dropshell
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace template registry system with git-backed template sources
All checks were successful
Build-Test-Publish / build (linux/amd64) (push) Successful in 43s
Details
Build-Test-Publish / build (linux/arm64) (push) Successful in 4m6s
Details

Browse Source
This commit is contained in:
j
2026-03-23 09:14:33 +13:00
parent 208433c436
commit 5217ad42d3
15 changed files with 447 additions and 1457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
source/src/commands/destroy.cpp
View File

@@ -79,6 +79,8 @@ namespace dropshell
LocalServiceInfo service_info;
service_info = get_service_info(server, service);
bool service_valid = SIvalid(service_info);
if (service_valid)
gTemplateManager().pull_for_template(service_info.template_name);
if (!service_valid)
warning << "No valid service definition found for " << service << std::endl;

3
source/src/commands/help.cpp
View File

@@ -158,7 +158,8 @@ int help_handler(const CommandContext& ctx) {
show_command("create-template");
info << std::endl;
show_command("validate-template");
show_command("publish-template");
info << std::endl;
show_command("pull");
}
return 0;
}

3
source/src/commands/install.cpp
View File

@@ -90,6 +90,9 @@ namespace dropshell
maketitle("Installing " + service + " (" + service_info.template_name + ") on " + server);
// Pull latest template from git if applicable
gTemplateManager().pull_for_template(service_info.template_name);
// Check if template exists
template_info tinfo = gTemplateManager().get_template_info(service_info.template_name);
if (!tinfo.is_set())

2
source/src/commands/list-templates.cpp
View File

@@ -27,7 +27,7 @@ struct ListTemplatesCommandRegister {
"list-templates",
"List all available templates and their sources.",
R"(
List all available templates from configured local paths and registries.
List all available templates from template_paths.json sources.
list-templates Show template names and where they come from.
)"
});

601
source/src/commands/publish-template.cpp
View File

@@ -1,601 +0,0 @@
#include "command_registry.hpp"
#include "config.hpp"
#include "templates.hpp"
#include "utils/utils.hpp"
#include "utils/directories.hpp"
#include "utils/hash.hpp"
#include "shared_commands.hpp"
#include <iostream>
#include <sstream>
#include <filesystem>
#include <fstream>
#include <ctime>
#include <iomanip>
#include <cstdlib>
#include <algorithm>
namespace dropshell {
void publish_template_autocomplete(const CommandContext& ctx);
int publish_template_handler(const CommandContext& ctx);
static std::vector<std::string> publish_template_name_list = {"publish-template", "publish"};
// Static registration
struct PublishTemplateCommandRegister {
PublishTemplateCommandRegister() {
CommandRegistry::instance().register_command({
publish_template_name_list,
publish_template_handler,
publish_template_autocomplete,
false, // hidden
true, // requires_config
true, // requires_install
1, // min_args
3, // max_args (--all + optional registry + directory)
"publish-template [--all] [REGISTRY] DIRECTORY",
"Publish a template to a template registry.",
R"HELP(
Publishes a template directory to a template registry.
Usage:
ds publish-template DIRECTORY
ds publish-template REGISTRY_NAME DIRECTORY
ds publish-template --all DIRECTORY
ds publish-template --all REGISTRY_NAME DIRECTORY
Arguments:
--all Publish all templates in subdirectories of DIRECTORY.
Only publishes subdirectories containing template_info.env
REGISTRY_NAME Optional. Name of the template registry to publish to.
If not specified, uses the first registry with a token.
DIRECTORY Path to the template directory (or parent directory with --all).
The template is validated before publishing. Two tags are created:
- YYYYMMDD (e.g., 20251228)
- latest
Authentication:
Token is resolved in this order:
1. Token configured in dropshell.json for the registry
2. SOS_WRITE_TOKEN environment variable (if single registry defined)
Requirements:
- Template must pass validation
- Valid authentication token
- curl must be available
Example:
ds publish-template ./my-template
ds publish-template main ./my-template
ds publish-template --all ./templates-dir
SOS_WRITE_TOKEN=xxx ds publish-template ./my-template
)HELP"
});
}
} publish_template_command_register;
void publish_template_autocomplete(const CommandContext& ctx) {
if (ctx.args.size() == 0) {
rawout << "--all" << std::endl;
// List registry names
std::vector<tRegistryEntry> registries = gConfig().get_template_registry_urls();
for (const auto& reg : registries) {
if (!reg.token.empty()) {
rawout << reg.name << std::endl;
}
}
}
}
// Get current date as YYYYMMDD (UTC)
static std::string get_date_tag() {
auto now = std::time(nullptr);
auto tm = *std::gmtime(&now);
std::ostringstream oss;
oss << std::put_time(&tm, "%Y%m%d");
return oss.str();
}
// Create tarball of template
static bool create_tarball(const std::string& template_dir, const std::string& tarball_path) {
std::filesystem::path dir_path(template_dir);
std::string template_name = dir_path.filename().string();
std::string parent_dir = dir_path.parent_path().string();
std::string cmd = "tar -czf \"" + tarball_path + "\" -C \"" + parent_dir + "\" \"" + template_name + "\" 2>/dev/null";
int ret = system(cmd.c_str());
return ret == 0 && std::filesystem::exists(tarball_path);
}
// Run curl and capture output
static bool run_curl(const std::string& cmd, std::string& output, int& http_code) {
// Append -w to get HTTP status code
std::string full_cmd = cmd + " -w '\\nHTTP_CODE:%{http_code}' 2>&1";
FILE* pipe = popen(full_cmd.c_str(), "r");
if (!pipe) return false;
char buffer[512];
output.clear();
while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
output += buffer;
}
int ret = pclose(pipe);
// Extract HTTP code from output
size_t code_pos = output.find("HTTP_CODE:");
if (code_pos != std::string::npos) {
http_code = std::stoi(output.substr(code_pos + 10));
output = output.substr(0, code_pos);
// Remove trailing newline
while (!output.empty() && (output.back() == '\n' || output.back() == '\r')) {
output.pop_back();
}
} else {
http_code = 0;
}
return ret == 0 || http_code == 200;
}
// Check if file already exists on server by hash
static bool check_file_exists(const std::string& server_url, const std::string& file_hash) {
std::string cmd = "curl -s \"" + server_url + "/exists/" + file_hash + "\"";
std::string output;
int http_code;
if (!run_curl(cmd, output, http_code)) {
return false;
}
// Response is JSON: {"exists": true} or {"exists": false}
return output.find("\"exists\":true") != std::string::npos ||
output.find("\"exists\": true") != std::string::npos;
}
// Get unpacked hash for a template:tag from server metadata
// Returns empty string if not found or error
static std::string get_remote_unpacked_hash(const std::string& server_url, const std::string& labeltag) {
std::string cmd = "curl -s \"" + server_url + "/meta/" + labeltag + "\"";
std::string output;
int http_code;
if (!run_curl(cmd, output, http_code) || http_code != 200) {
return "";
}
// Parse unpackedhash from JSON response
// Look for "unpackedhash":"<hash>" or "unpackedhash": "<hash>"
size_t pos = output.find("\"unpackedhash\"");
if (pos == std::string::npos) {
return "";
}
// Find the colon and opening quote
pos = output.find(':', pos);
if (pos == std::string::npos) return "";
pos = output.find('"', pos);
if (pos == std::string::npos) return "";
pos++; // skip opening quote
size_t end = output.find('"', pos);
if (end == std::string::npos) return "";
return output.substr(pos, end - pos);
}
// Upload a new file to the server
static bool upload_file(const std::string& server_url, const std::string& token,
const std::string& file_path, const std::vector<std::string>& labeltags,
const std::string& unpacked_hash) {
// Build labeltags JSON array
std::ostringstream labeltags_json;
labeltags_json << "[";
for (size_t i = 0; i < labeltags.size(); i++) {
if (i > 0) labeltags_json << ",";
labeltags_json << "\"" << labeltags[i] << "\"";
}
labeltags_json << "]";
// Build metadata JSON
std::ostringstream metadata;
metadata << "{\"labeltags\":" << labeltags_json.str()
<< ",\"unpackedhash\":\"" << unpacked_hash << "\""
<< ",\"description\":\"Published by dropshell\"}";
// Build curl command for multipart upload
std::string cmd = "curl -s -X PUT "
"-H \"Authorization: Bearer " + token + "\" "
"-F \"file=@" + file_path + "\" "
"-F 'metadata=" + metadata.str() + "' "
"\"" + server_url + "/upload\"";
std::string output;
int http_code;
if (!run_curl(cmd, output, http_code)) {
error << "Upload failed: " << output << std::endl;
return false;
}
if (http_code != 200) {
error << "Upload failed with HTTP " << http_code << ": " << output << std::endl;
return false;
}
return true;
}
// Update metadata for an existing file
static bool update_metadata(const std::string& server_url, const std::string& token,
const std::string& file_hash, const std::vector<std::string>& labeltags,
const std::string& unpacked_hash) {
// Build labeltags JSON array
std::ostringstream labeltags_json;
labeltags_json << "[";
for (size_t i = 0; i < labeltags.size(); i++) {
if (i > 0) labeltags_json << ",";
labeltags_json << "\"" << labeltags[i] << "\"";
}
labeltags_json << "]";
// Build metadata JSON
std::ostringstream metadata;
metadata << "{\"labeltags\":" << labeltags_json.str()
<< ",\"unpackedhash\":\"" << unpacked_hash << "\""
<< ",\"description\":\"Published by dropshell\"}";
// Build the full request body
std::ostringstream body;
body << "{\"hash\":\"" << file_hash << "\",\"metadata\":" << metadata.str() << "}";
// Build curl command for JSON update
std::string cmd = "curl -s -X PUT "
"-H \"Authorization: Bearer " + token + "\" "
"-H \"Content-Type: application/json\" "
"-d '" + body.str() + "' "
"\"" + server_url + "/update\"";
std::string output;
int http_code;
if (!run_curl(cmd, output, http_code)) {
error << "Update failed: " << output << std::endl;
return false;
}
if (http_code != 200) {
error << "Update failed with HTTP " << http_code << ": " << output << std::endl;
return false;
}
return true;
}
// Publish a file to the registry (upload or update metadata if exists)
static bool publish_to_registry(const std::string& server_url, const std::string& token,
const std::string& file_path, const std::vector<std::string>& labeltags,
const std::string& unpacked_hash) {
// Calculate SHA256 hash of the tarball for deduplication
std::string file_hash = hash_file(file_path);
if (file_hash.empty()) {
error << "Failed to calculate file hash" << std::endl;
return false;
}
// Check if file already exists on server
bool exists = check_file_exists(server_url, file_hash);
if (exists) {
// File exists, just update metadata (adds new labeltags)
return update_metadata(server_url, token, file_hash, labeltags, unpacked_hash);
} else {
// File doesn't exist, upload it
return upload_file(server_url, token, file_path, labeltags, unpacked_hash);
}
}
// Check if a directory is a valid template (has template_info.env at root)
static bool is_valid_template_dir(const std::filesystem::path& dir_path) {
return std::filesystem::exists(dir_path / "template_info.env");
}
// Publish a single template directory
// Returns: 0 = success, 1 = error, 2 = skipped (not a template), 3 = unchanged (already up to date)
static int publish_single_template(const std::string& template_dir, const std::string& server_url,
const std::string& token, bool quiet = false, bool skip_if_unchanged = false) {
std::filesystem::path dir_path(template_dir);
std::string template_name = dir_path.filename().string();
if (!quiet) {
maketitle("Publishing template: " + template_name);
info << "Directory: " << template_dir << std::endl;
std::cout << std::endl;
}
// Validate template
if (!quiet) info << "=== Validating Template ===" << std::endl;
if (!template_manager::test_template(template_dir)) {
error << "Template '" << template_name << "' validation failed." << std::endl;
if (!quiet) info << "Run: ds validate-template " << template_dir << std::endl;
return 1;
}
if (!quiet) info << " ✓ Template is valid" << std::endl;
if (!quiet) std::cout << std::endl;
// Calculate directory hash
if (!quiet) info << "=== Calculating Hash ===" << std::endl;
std::string unpacked_hash = hash_directory_recursive(template_dir);
if (unpacked_hash.empty()) {
error << "Failed to calculate directory hash for " << template_name << std::endl;
return 1;
}
if (!quiet) info << " Hash: " << unpacked_hash << std::endl;
// Check if unchanged (compare with remote :latest)
if (skip_if_unchanged) {
std::string remote_hash = get_remote_unpacked_hash(server_url, template_name + ":latest");
if (!remote_hash.empty() && remote_hash == unpacked_hash) {
if (!quiet) {
info << " ✓ Unchanged (matches remote)" << std::endl;
std::cout << std::endl;
}
return 3; // unchanged
}
}
if (!quiet) std::cout << std::endl;
// Create temp directory and tarball
std::string temp_dir = std::filesystem::temp_directory_path().string() + "/dropshell-publish-" + std::to_string(getpid());
std::filesystem::create_directories(temp_dir);
// Cleanup on exit
struct TempDirCleaner {
std::string path;
~TempDirCleaner() {
std::filesystem::remove_all(path);
}
} cleaner{temp_dir};
if (!quiet) info << "=== Creating Package ===" << std::endl;
std::string tarball_path = temp_dir + "/" + template_name + ".tgz";
if (!create_tarball(template_dir, tarball_path)) {
error << "Failed to create tarball for " << template_name << std::endl;
return 1;
}
auto file_size = std::filesystem::file_size(tarball_path);
if (!quiet) info << " Created " << template_name << ".tgz (" << (file_size / 1024) << " KB)" << std::endl;
if (!quiet) std::cout << std::endl;
// Publish with tags
if (!quiet) info << "=== Publishing to Registry ===" << std::endl;
std::string date_tag = get_date_tag();
std::vector<std::string> labeltags = {
template_name + ":" + date_tag,
template_name + ":latest"
};
if (!quiet) info << " Tags: " << date_tag << ", latest" << std::endl;
if (!quiet) info << " Uploading..." << std::endl;
if (!publish_to_registry(server_url, token, tarball_path, labeltags, unpacked_hash)) {
error << "Failed to publish template " << template_name << std::endl;
return 1;
}
if (!quiet) {
info << " ✓ Published successfully" << std::endl;
std::cout << std::endl;
}
return 0;
}
int publish_template_handler(const CommandContext& ctx) {
bool publish_all = false;
std::string registry_name;
std::string target_dir;
// Parse arguments - check for --all flag
std::vector<std::string> args;
for (const auto& arg : ctx.args) {
if (arg == "--all" || arg == "-a" || arg == "all") {
publish_all = true;
} else {
args.push_back(arg);
}
}
// Parse remaining arguments
if (args.size() == 1) {
target_dir = args[0];
} else if (args.size() == 2) {
registry_name = args[0];
target_dir = args[1];
} else if (args.empty()) {
error << "Usage: ds publish-template [--all] [REGISTRY] DIRECTORY" << std::endl;
return 1;
} else {
error << "Too many arguments" << std::endl;
error << "Usage: ds publish-template [--all] [REGISTRY] DIRECTORY" << std::endl;
return 1;
}
// Resolve directory to absolute path
std::filesystem::path dir_path(target_dir);
if (dir_path.is_relative()) {
dir_path = std::filesystem::current_path() / dir_path;
}
dir_path = std::filesystem::canonical(dir_path);
target_dir = dir_path.string();
// Check directory exists
if (!std::filesystem::exists(dir_path) || !std::filesystem::is_directory(dir_path)) {
error << "Directory not found: " << target_dir << std::endl;
return 1;
}
// Find registry and token
std::vector<tRegistryEntry> registries = gConfig().get_template_registry_urls();
tRegistryEntry* selected_registry = nullptr;
std::string effective_token;
if (registry_name.empty()) {
// Find first registry with a token
for (auto& reg : registries) {
if (!reg.token.empty()) {
selected_registry = &reg;
effective_token = reg.token;
break;
}
}
// Fallback: if no registry has a token, but there's exactly one registry
// and SOS_WRITE_TOKEN is set, use that
if (!selected_registry && registries.size() == 1) {
const char* env_token = std::getenv("SOS_WRITE_TOKEN");
if (env_token && env_token[0] != '\0') {
selected_registry = &registries[0];
effective_token = env_token;
info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl;
}
}
if (!selected_registry) {
error << "No template registry with a token found" << std::endl;
info << "Either:" << std::endl;
info << " - Add a token to a registry in dropshell.json" << std::endl;
info << " - Set SOS_WRITE_TOKEN environment variable (if only one registry defined)" << std::endl;
return 1;
}
} else {
// Find registry by name
for (auto& reg : registries) {
if (reg.name == registry_name) {
selected_registry = &reg;
break;
}
}
if (!selected_registry) {
error << "Registry not found: " << registry_name << std::endl;
return 1;
}
// Use token from config, or fall back to SOS_WRITE_TOKEN env var
if (!selected_registry->token.empty()) {
effective_token = selected_registry->token;
} else {
const char* env_token = std::getenv("SOS_WRITE_TOKEN");
if (env_token && env_token[0] != '\0') {
effective_token = env_token;
info << "Using SOS_WRITE_TOKEN environment variable for authentication" << std::endl;
} else {
error << "Registry '" << registry_name << "' does not have a token configured" << std::endl;
info << "Set token in dropshell.json or SOS_WRITE_TOKEN environment variable" << std::endl;
return 1;
}
}
}
// Build server URL (ensure https://)
std::string server_url = selected_registry->url;
if (server_url.substr(0, 8) != "https://" && server_url.substr(0, 7) != "http://") {
server_url = "https://" + server_url;
}
// Remove trailing slash if present
if (!server_url.empty() && server_url.back() == '/') {
server_url.pop_back();
}
if (publish_all) {
// Publish all templates in subdirectories
maketitle("Publishing all templates");
info << "Parent directory: " << target_dir << std::endl;
info << "Registry: " << selected_registry->name << " (" << server_url << ")" << std::endl;
std::cout << std::endl;
// Find all subdirectories that are valid templates
std::vector<std::filesystem::path> template_dirs;
for (const auto& entry : std::filesystem::directory_iterator(dir_path)) {
if (entry.is_directory() && is_valid_template_dir(entry.path())) {
template_dirs.push_back(entry.path());
}
}
// Sort alphabetically
std::sort(template_dirs.begin(), template_dirs.end());
if (template_dirs.empty()) {
error << "No valid templates found in " << target_dir << std::endl;
info << "Templates must have template_info.env" << std::endl;
return 1;
}
info << "Found " << template_dirs.size() << " template(s) to publish:" << std::endl;
for (const auto& tdir : template_dirs) {
info << " - " << tdir.filename().string() << std::endl;
}
std::cout << std::endl;
int success_count = 0;
int unchanged_count = 0;
int fail_count = 0;
std::vector<std::string> failed_templates;
for (const auto& tdir : template_dirs) {
int result = publish_single_template(tdir.string(), server_url, effective_token, false, true);
if (result == 0) {
success_count++;
} else if (result == 3) {
unchanged_count++;
} else {
fail_count++;
failed_templates.push_back(tdir.filename().string());
}
}
// Summary
std::cout << std::endl;
maketitle("Publish Summary");
if (success_count > 0) {
info << "Published: " << success_count << " template(s)" << std::endl;
}
if (unchanged_count > 0) {
info << "Unchanged: " << unchanged_count << " template(s)" << std::endl;
}
if (fail_count > 0) {
error << "Failed: " << fail_count << " template(s)" << std::endl;
for (const auto& name : failed_templates) {
error << " - " << name << std::endl;
}
return 1;
}
return 0;
} else {
// Single template publish
info << "Registry: " << selected_registry->name << " (" << server_url << ")" << std::endl;
int result = publish_single_template(target_dir, server_url, effective_token, false);
if (result == 0) {
std::string template_name = dir_path.filename().string();
maketitle("Publish Complete");
info << "Template '" << template_name << "' published successfully!" << std::endl;
info << "Tags: " << get_date_tag() << ", latest" << std::endl;
info << "URL: " << server_url << "/" << template_name << ":latest" << std::endl;
}
return result;
}
}
} // namespace dropshell

44
source/src/commands/pull.cpp Normal file
View File

@@ -0,0 +1,44 @@
#include "command_registry.hpp"
#include "templates.hpp"
#include "utils/output.hpp"
namespace dropshell {
int pull_handler(const CommandContext& ctx);
static std::vector<std::string> pull_name_list = {"pull"};
struct PullCommandRegister {
PullCommandRegister() {
CommandRegistry::instance().register_command({
pull_name_list,
pull_handler,
nullptr,
false, // hidden
true, // requires_config
false, // requires_install
0, // min_args
0, // max_args
"pull",
"Pull/clone all git-backed template sources.",
R"(
Pull the latest changes for all git-backed template sources
defined in template_paths.json.
If a source directory doesn't exist and has a git URL, it will be cloned.
If it exists with a .git directory, it will be pulled.
Local-only sources (no git URL) are skipped.
)"
});
}
} pull_command_register;
int pull_handler(const CommandContext& ctx) {
if (!gTemplateManager().is_loaded()) {
error << "Template manager not loaded" << std::endl;
return 1;
}
return gTemplateManager().pull_all() ? 0 : 1;
}
} // namespace dropshell

2
source/src/commands/restoredata.cpp
View File

@@ -105,6 +105,8 @@ namespace dropshell
return 1;
}
// Pull latest template from git if applicable
gTemplateManager().pull_for_template(service_info.template_name);
if (!gTemplateManager().template_command_exists(service_info.template_name, "backup") ||
!gTemplateManager().template_command_exists(service_info.template_name, "restore"))

119
source/src/config.cpp
View File

@@ -46,8 +46,6 @@ bool config::load_config() { // load json config file.
// Validate the config format - check for required and supported fields
std::set<std::string> allowed_fields = {
"server_definition_paths",
"template_local_paths",
"template_registries",
"backups_path",
"log_level",
"disabled_servers"
@@ -55,25 +53,16 @@ bool config::load_config() { // load json config file.
std::set<std::string> deprecated_fields = {
"template_registry_URLs",
"template_upload_token"
"template_upload_token",
"template_registries",
"template_local_paths"
};
// Check for deprecated fields
for (const auto& field : deprecated_fields) {
if (mConfig.contains(field)) {
error << "Config file contains deprecated field '" << field << "'" << std::endl;
error << "Please update your config file to the new format." << std::endl;
if (field == "template_registry_URLs") {
error << "Replace 'template_registry_URLs' with 'template_registries' using the format:" << std::endl;
error << " \"template_registries\": [" << std::endl;
error << " {" << std::endl;
error << " \"name\": \"main\"," << std::endl;
error << " \"url\": \"https://templates.dropshell.app\"," << std::endl;
error << " \"token\": \"\"" << std::endl;
error << " }" << std::endl;
error << " ]" << std::endl;
}
return false;
warning << "Config file contains deprecated field '" << field << "' - ignoring it." << std::endl;
mConfig.erase(field);
}
}
@@ -93,24 +82,6 @@ bool config::load_config() { // load json config file.
}
}
// Validate template_registries format if present
if (mConfig.contains("template_registries")) {
if (!mConfig["template_registries"].is_array()) {
error << "'template_registries' must be an array" << std::endl;
return false;
}
for (const auto& registry : mConfig["template_registries"]) {
if (!registry.is_object()) {
error << "Each registry in 'template_registries' must be an object" << std::endl;
return false;
}
if (!registry.contains("name") || !registry.contains("url")) {
error << "Each registry must have 'name' and 'url' fields" << std::endl;
return false;
}
}
}
// Validate log_level if present
if (mConfig.contains("log_level")) {
if (!mConfig["log_level"].is_string()) {
@@ -131,15 +102,6 @@ bool config::load_config() { // load json config file.
return true;
}
void _append(std::vector<std::string> & a, const std::vector<std::string> & b) {
if (b.empty())
return;
if (a.empty())
a = b;
else
a.insert(std::end(a), std::begin(b), std::end(b));
}
bool config::save_config()
{
std::string config_path = localfile::dropshell_json();
@@ -160,16 +122,6 @@ bool config::save_config()
mConfig["server_definition_paths"] = {
dropshell_base + "/servers"
};
mConfig["template_local_paths"] = {
dropshell_base + "/local_templates"
};
mConfig["template_registries"] = nlohmann::json::array({
nlohmann::json::object({
{"name", "main"},
{"url", "https://templates.dropshell.app"},
{"token", ""}
})
});
mConfig["backups_path"] = dropshell_base + "/backups";
mConfig["log_level"] = "info"; // Default log level
}
@@ -187,9 +139,7 @@ bool config::save_config()
bool config::create_aux_directories()
{
std::vector<std::string> paths;
_append(paths, get_local_template_paths());
_append(paths, get_local_server_definition_paths());
std::vector<std::string> paths = get_local_server_definition_paths();
for (auto & p : paths)
if (!std::filesystem::exists(p))
{
@@ -209,27 +159,6 @@ bool config::is_agent_installed()
return std::filesystem::exists(localfile::bb64());
}
std::vector<tRegistryEntry> config::get_template_registry_urls() {
nlohmann::json template_registries = mConfig["template_registries"];
std::vector<tRegistryEntry> registries;
for (auto &registry : template_registries) {
if (registry.is_object() && !registry.empty())
registries.push_back(tRegistryEntry(registry));
}
return registries;
}
std::vector<std::string> config::get_local_template_paths()
{
nlohmann::json template_local_paths = mConfig["template_local_paths"];
std::vector<std::string> paths;
for (auto &path : template_local_paths) {
if (path.is_string() && !path.empty())
paths.push_back(path);
}
return paths;
}
std::vector<std::string> config::get_local_server_definition_paths() {
nlohmann::json server_definition_paths = mConfig["server_definition_paths"];
std::vector<std::string> paths;
@@ -248,14 +177,6 @@ std::string config::get_server_create_path()
return paths[0];
}
std::string config::get_template_create_path()
{
std::vector<std::string> paths = get_local_template_paths();
if (paths.empty())
return "";
return paths[0];
}
std::string config::get_backups_path()
{
nlohmann::json backups_path = mConfig["backups_path"];
@@ -267,34 +188,6 @@ std::string config::get_backups_path()
return "";
}
dropshell::tRegistryEntry::tRegistryEntry(nlohmann::json json)
{
valid = false;
if (json.is_object() && !json.empty()) {
for (auto &[key, value] : json.items()) {
if (value.is_string() && !value.empty())
switch (switchhash(key.c_str())) {
case switchhash("name"):
name = value;
break;
case switchhash("url"):
url = value;
break;
case switchhash("token"):
token = value;
break;
default:
break;
}
}
valid = (!url.empty()&&!name.empty()); // token can be empty.
}
}
tRegistryEntry::~tRegistryEntry()
{
}
std::string config::get_log_level() const
{
if (!mIsConfigSet || !mConfig.contains("log_level"))

16
source/src/config.hpp
View File

@@ -8,19 +8,6 @@
namespace dropshell {
class tRegistryEntry {
public:
tRegistryEntry(nlohmann::json json);
~tRegistryEntry();
public:
std::string name;
std::string url;
std::string token;
bool valid;
};
class config {
public:
config();
@@ -33,12 +20,9 @@ class config {
bool is_config_set() const;
static bool is_agent_installed();
std::vector<tRegistryEntry> get_template_registry_urls();
std::vector<std::string> get_local_template_paths();
std::vector<std::string> get_local_server_definition_paths();
std::string get_server_create_path();
std::string get_template_create_path();
std::string get_backups_path();
std::string get_log_level() const;

79
source/src/services.cpp
View File

@@ -23,7 +23,7 @@ namespace dropshell
!service_info.user.empty();
}
std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name, bool skip_update)
std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name)
{
std::vector<LocalServiceInfo> services;
@@ -49,7 +49,7 @@ namespace dropshell
std::string dirname = entry.path().filename().string();
if (dirname.empty() || dirname[0] == '.' || dirname[0] == '_')
continue;
auto service = get_service_info(server_name, dirname, skip_update);
auto service = get_service_info(server_name, dirname);
if (!service.local_service_path.empty())
services.push_back(service);
else
@@ -72,7 +72,7 @@ namespace dropshell
return it->second == "true";
}
LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool skip_update)
LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name)
{
if (server_name.empty() || service_name.empty())
@@ -114,15 +114,12 @@ namespace dropshell
}
service.template_name = it->second;
tinfo = gTemplateManager().get_template_info(service.template_name,skip_update);
tinfo = gTemplateManager().get_template_info(service.template_name);
if (!tinfo.is_set())
{
// Template not found - this means it's not available locally OR from registry
error << "Template '" << service.template_name << "' not found locally or in registry" << std::endl;
error << "Template '" << service.template_name << "' not found in local template paths" << std::endl;
return LocalServiceInfo();
}
// Template is available (either locally or downloaded from registry)
service.local_template_path = tinfo.local_template_path();
{ // set the user.
@@ -208,70 +205,4 @@ namespace dropshell
return backups;
}
// bool get_all_service_env_vars(const std::string &server_name, const std::string &service_name, ordered_env_vars &all_env_vars)
// {
// clear_vars(all_env_vars);
// if (localpath::service(server_name, service_name).empty() || !fs::exists(localpath::service(server_name, service_name)))
// {
// error << "Service not found: " << service_name << " on server " << server_name << std::endl;
// return false;
// }
// // Lambda function to load environment variables from a file
// auto load_env_file = [&all_env_vars](const std::string &file)
// {
// if (!file.empty() && std::filesystem::exists(file))
// {
// ordered_env_vars env_vars;
// envmanager env_manager(file);
// env_manager.load();
// env_manager.get_all_variables(env_vars);
// merge_vars(all_env_vars, env_vars);
// }
// else
// warning << "Expected environment file not found: " << file << std::endl;
// };
// // add in some simple variables first, as others below may depend on/use these in bash.
// set_var(all_env_vars, "SERVER", server_name);
// set_var(all_env_vars, "SERVICE", service_name);
// set_var(all_env_vars, "DOCKER_CLI_HINTS", "false"); // turn off docker junk.
// // Load environment files
// load_env_file(localfile::service_env(server_name, service_name));
// std::string template_name = get_var(all_env_vars, "TEMPLATE");
// if (template_name.empty())
// {
// error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
// return false;
// }
// auto tinfo = gTemplateManager().get_template_info(template_name, true); // skip updates.
// if (!tinfo.is_set())
// {
// // Template is not available locally or from registry
// error << "Template '" << template_name << "' not found locally or in registry" << std::endl;
// return false;
// }
// ASSERT(std::filesystem::exists(tinfo.local_template_info_env_path()));
// load_env_file(tinfo.local_template_info_env_path());
// std::string user = get_var(all_env_vars, "SSH_USER");
// if (user.empty())
// {
// error << "SSH_USER variable not defined in service " << service_name << " on server " << server_name << std::endl;
// info << "This variable definition is always required, and usually set in the "<<filenames::service_env << " file." << std::endl;
// info << "Please check " << localfile::service_env(server_name, service_name) << std::endl;
// return false;
// }
// // more additional, these depend on others above.
// set_var(all_env_vars, "CONFIG_PATH", remotepath(server_name, user).service_config(service_name));
// set_var(all_env_vars, "AGENT_PATH", remotepath(server_name, user).agent());
// return true;
// }
} // namespace dropshell

7
source/src/services.hpp
View File

@@ -4,7 +4,6 @@
#include <string>
#include <vector>
#include <set>
//#include <map>
#include "utils/ordered_env.hpp"
namespace dropshell {
@@ -18,15 +17,13 @@ namespace dropshell {
bool requires_host_root;
bool requires_docker;
bool requires_docker_root;
//bool service_template_hash_match;
};
bool SIvalid(const LocalServiceInfo& service_info);
// if skip_update, don't check for updates to the service template.
std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name, bool skip_update=false);
std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name);
LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool skip_update=false);
LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name);
std::set<std::string> get_used_commands(const std::string& server_name, const std::string& service_name);
// list all backups for a given service (across all servers)

862
source/src/templates.cpp
View File

File diff suppressed because it is too large Load Diff

89
source/src/templates.hpp
View File

@@ -1,26 +1,31 @@
#pragma once
#include <string>
#include <vector>
#include <filesystem>
#include <memory>
#include <map>
#include <set>
#include "config.hpp"
#define JSON_INLINE_ALL
#include <nlohmann/json.hpp>
namespace dropshell {
typedef enum template_source_type {
TEMPLATE_SOURCE_TYPE_LOCAL,
TEMPLATE_SOURCE_TYPE_REGISTRY,
TEMPLATE_SOURCE_NOT_SET
} template_source_type;
// Represents one entry from template_paths.json
struct TemplateSource {
std::string local_path; // Git checkout root or local directory
std::string git_url; // Optional, empty if local-only
};
// Represents one resolved template
struct ResolvedTemplate {
std::filesystem::path template_dir;
size_t source_index; // Index into mSources
};
class template_info {
public:
template_info() : mIsSet(false) {}
template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool skip_update);
template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path);
virtual ~template_info() {}
bool is_set() const { return mIsSet; }
std::string name() const { return mTemplateName; }
@@ -33,55 +38,11 @@ class template_info {
private:
std::string mTemplateName;
std::string mLocationID;
std::filesystem::path mTemplateLocalPath; // source or cache.
std::filesystem::path mTemplateLocalPath;
bool mTemplateValid;
bool mIsSet;
};
class template_source_interface {
public:
virtual ~template_source_interface() {}
virtual std::set<std::string> get_template_list() = 0;
virtual bool has_template(const std::string& template_name) = 0;
virtual template_info get_template_info(const std::string& template_name, bool skip_update=false) = 0;
virtual bool template_command_exists(const std::string& template_name,const std::string& command) = 0;
virtual std::string get_description() = 0;
};
class template_source_registry : public template_source_interface {
public:
template_source_registry(tRegistryEntry registry) : mRegistry(registry) {}
~template_source_registry() {}
std::set<std::string> get_template_list();
bool has_template(const std::string& template_name);
template_info get_template_info(const std::string& template_name, bool skip_update=false);
bool template_command_exists(const std::string& template_name,const std::string& command);
std::string get_description() { return "Registry: " + mRegistry.name + " (" + mRegistry.url + ")"; }
private:
std::filesystem::path get_cache_dir();
private:
tRegistryEntry mRegistry;
std::vector<nlohmann::json> mTemplates; // cached list.
};
class template_source_local : public template_source_interface {
public:
template_source_local(std::string local_path) : mLocalPath(local_path) {}
~template_source_local() {}
std::set<std::string> get_template_list();
bool has_template(const std::string& template_name);
template_info get_template_info(const std::string& template_name, bool skip_update=false);
bool template_command_exists(const std::string& template_name,const std::string& command);
std::string get_description() { return "Local: " + mLocalPath.string(); }
private:
std::filesystem::path mLocalPath;
};
class template_manager {
public:
template_manager() : mLoaded(false) {}
@@ -89,7 +50,7 @@ class template_manager {
std::set<std::string> get_template_list() const;
bool has_template(const std::string& template_name) const;
template_info get_template_info(const std::string& template_name, bool skip_update=false) const; // skip_update = don't check for updates.
template_info get_template_info(const std::string& template_name) const;
bool template_command_exists(const std::string& template_name, const std::string& command) const;
bool create_template(const std::string& template_name) const;
@@ -105,16 +66,26 @@ class template_manager {
bool is_loaded() const { return mLoaded; }
int get_source_count() const { return mSources.size(); }
// Git pull operations
bool pull_all();
bool pull_for_template(const std::string& template_name);
private:
static bool required_file(std::string path, std::string template_name);
template_source_interface* get_source(const std::string& template_name) const;
void resolve_templates();
void resolve_source(size_t source_index);
std::vector<std::string> parse_list_file(const std::string& local_path) const;
std::vector<std::string> discover_templates(const std::string& local_path) const;
void write_list_file(const std::string& local_path, const std::vector<std::string>& template_dirs) const;
bool git_pull_source(const TemplateSource& source);
bool git_clone_source(const TemplateSource& source);
private:
bool mLoaded;
mutable std::vector<std::unique_ptr<template_source_interface>> mSources;
std::vector<TemplateSource> mSources;
std::map<std::string, ResolvedTemplate> mTemplateMap;
};
template_manager & gTemplateManager();
} // namespace dropshell

6
source/src/utils/directories.cpp
View File

@@ -128,11 +128,6 @@ namespace dropshell
return dropshell_dir() + "/temp_files";
}
std::string template_cache()
{
return dropshell_dir() + "/template_cache";
}
std::string template_example()
{
return agent_local() + "/template_example";
@@ -145,7 +140,6 @@ namespace dropshell
dropshell_dir(),
agent_local(),
agent_remote(),
template_cache(),
backups(),
temp_files()};
for (auto &p : gConfig().get_local_server_definition_paths())

11
source/src/utils/directories.hpp
View File

@@ -20,14 +20,6 @@ namespace dropshell {
// |-- agent-remote
// |-- (remote agent files)
// |-- temp_files
// |-- template_cache
// | |-- <template_name>.json
// | |-- <template_name>
// | |-- (...script files...)
// | |-- template_info.env
// | |-- config
// | |-- service.env
// | |-- (...other service config files...)
// backups_path
// |-- katie-_-squashkiwi-_-squashkiwi-test-_-2025-04-28_21-23-59.tgz
@@ -49,6 +41,8 @@ namespace dropshell {
static const std::string server_json = "server.json";
static const std::string dropshell_json = "dropshell.json";
static const std::string ds_run = "ds_run.sh";
static const std::string template_paths_json = "template_paths.json";
static const std::string dropshell_templates_list = "dropshell-templates.list";
} // namespace filenames.
namespace localfile {
@@ -77,7 +71,6 @@ namespace dropshell {
std::string backups();
std::string temp_files();
std::string template_cache();
bool create_directories();
} // namespace local