public/dropshell
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Major refactor

Browse Source
This commit is contained in:
j
2025-12-30 08:43:06 +13:00
parent a9ccdedf89
commit b0d11eb08c
10 changed files with 84 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
source/src/commands/create-service.cpp
View File

@@ -156,13 +156,13 @@ namespace dropshell
// copy the template config files to the service directory
recursive_copy(tinfo.local_template_path() / "config", service_dir);
// append TEMPLATE_HASH to the .template_info.env file
std::string template_info_env_file = localfile::template_info_env(server_name,service_name);
ASSERT(std::filesystem::exists(template_info_env_file), "Template info env file not found: " + template_info_env_file);
std::ofstream template_info_env_file_out(template_info_env_file, std::ios::app); // append to the file.
template_info_env_file_out << std::endl << "TEMPLATE_HASH=" << tinfo.hash() << std::endl;
template_info_env_file_out.close();
// append TEMPLATE to the service.env file
std::string service_env_file = localfile::service_env(server_name, service_name);
ASSERT(std::filesystem::exists(service_env_file),"service.env not found: "+ service_env_file);
std::ofstream ofs(service_env_file, std::ios::app);
ofs << std::endl << "# TEMPLATE set by dropshell on creation. Important this does not change." << std::endl;
ofs << "TEMPLATE=\"" << template_name <<"\"" << std::endl;
ofs.close();
// modify the SSH_USER to be nice.
// everything is created, so we can get the service info.
@@ -245,34 +245,6 @@ namespace dropshell
return true;
}
bool merge_updated_service_template(const std::string &server_name, const std::string &service_name)
{
LocalServiceInfo service_info = get_service_info(server_name, service_name);
ASSERT(SIvalid(service_info), "Service info is not valid for " + service_name + " on " + server_name);
template_info tinfo = gTemplateManager().get_template_info(service_info.template_name);
ASSERT(tinfo.is_set(), "Failed to load template " + service_info.template_name);
// copy across .template_info.env file
std::string template_info_env_file = tinfo.local_template_path() / "config" / filenames::template_info_env;
std::string target_template_info_env_file = localfile::template_info_env(server_name, service_name);
ASSERT(std::filesystem::exists(template_info_env_file), "Template service env file not found: " + template_info_env_file);
std::filesystem::remove(target_template_info_env_file);
std::filesystem::copy(template_info_env_file, target_template_info_env_file);
#pragma message("TODO: merge the template info env file")
// update hash in template info env file
// append TEMPLATE_HASH to the .template_info.env file
ASSERT(std::filesystem::exists(target_template_info_env_file), "Template info env file not found: " + target_template_info_env_file);
std::ofstream template_info_env_file_out(target_template_info_env_file, std::ios::app); // append to the file.
template_info_env_file_out << std::endl << "TEMPLATE_HASH=" << tinfo.hash() << std::endl;
template_info_env_file_out.close();
return true;
}
} // namespace shared_commands
} // namespace dropshell

22
source/src/commands/install.cpp
View File

@@ -93,23 +93,6 @@ namespace dropshell
shared_commands::uninstall_service(server_env, service);
}
if (!service_info.service_template_hash_match)
{
warning << "Service " << service << " is using an old template. Updating. " << std::endl;
if (!merge_updated_service_template(server_env.get_server_name(), service))
{
error << "Failed to merge updated service template. " << std::endl;
return false;
}
service_info = get_service_info(server_env.get_server_name(), service);
if (!SIvalid(service_info) || !service_info.service_template_hash_match)
{
error << "Merged updated service template, but it is still not valid. " << std::endl;
return false;
}
}
maketitle("Installing " + service + " (" + service_info.template_name + ") on " + server);
// Check if template exists
@@ -541,12 +524,11 @@ complete -F _dropshell_completions ds
// copy server.json across.
info << "Copying server.json to remote server... " <<std::flush;
shared_commands::rsync_file_to_remote(
localfile::server_json(server)),
remotepath(server.get_server_name(),user.user).server_json(),
localfile::server_json(server.get_server_name()),
remotefile(server.get_server_name(),user.user).server_json(),
server, false, user.user);
info << "done." << std::endl;
// run the agent installer. Can't use BB64 yet, as we're installing it on the remote server.
bool okay = execute_ssh_command(server.get_SSH_INFO(user.user), sCommand(agent_path, "agent-install.sh",{}), cMode::Defaults | cMode::NoBB64, nullptr);
if (!okay)

1
source/src/commands/shared_commands.hpp
View File

@@ -120,7 +120,6 @@ namespace dropshell
// defined in create-service.cpp
bool create_service(const std::string &server_name, const std::string &template_name, const std::string &service_name, std::string user_override="");
bool merge_updated_service_template(const std::string &server_name, const std::string &service_name);
} // namespace shared_commands
} // namespace dropshell

71
source/src/services.cpp
View File

@@ -14,7 +14,6 @@ namespace fs = std::filesystem;
namespace dropshell
{
#pragma message("TODO : Smart test that the service is fully valid.")
bool SIvalid(const LocalServiceInfo &service_info)
{
return !service_info.service_name.empty() &&
@@ -24,7 +23,7 @@ namespace dropshell
!service_info.user.empty();
}
std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name, bool fast)
std::vector<LocalServiceInfo> get_server_services_info(const std::string &server_name, bool skip_update)
{
std::vector<LocalServiceInfo> services;
@@ -50,7 +49,7 @@ namespace dropshell
std::string dirname = entry.path().filename().string();
if (dirname.empty() || dirname[0] == '.' || dirname[0] == '_')
continue;
auto service = get_service_info(server_name, dirname, fast);
auto service = get_service_info(server_name, dirname, skip_update);
if (!service.local_service_path.empty())
services.push_back(service);
else
@@ -73,7 +72,7 @@ namespace dropshell
return it->second == "true";
}
LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool fast)
LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool skip_update)
{
LocalServiceInfo service;
@@ -111,7 +110,7 @@ namespace dropshell
service.template_name = it->second;
}
template_info tinfo = gTemplateManager().get_template_info(service.template_name,fast);
template_info tinfo = gTemplateManager().get_template_info(service.template_name,skip_update);
if (!tinfo.is_set())
{
// Template not found - this means it's not available locally OR from registry
@@ -137,30 +136,6 @@ namespace dropshell
service.requires_docker = get_bool_variable(variables, "REQUIRES_DOCKER");
service.requires_docker_root = get_bool_variable(variables, "REQUIRES_DOCKER_ROOT");
{ // determine if the service template hash matches the template hash.
auto it = find_var(variables, "TEMPLATE_HASH");
service.service_template_hash_match = false;
if (it == variables.end())
{
// For backward compatibility with services created before hash mechanism
debug << "TEMPLATE_HASH not found in " << filenames::template_info_env << " for " << server_name << " - " << service.template_name
<< " (service may have been created before hash tracking was implemented)" << std::endl;
}
else if (tinfo.is_set())
{
std::string template_hash_str = it->second;
// Compare hash strings directly (migration from uint64_t to SHA256 string)
service.service_template_hash_match = (template_hash_str == tinfo.hash());
//debug << "Service template hash: " << service_template_hash << " == " << tinfo.hash() << std::endl;
}
else
{
// Template not available yet, can't check hash
debug << "Couldn't check template hash as the template info is not available (yet?)" << std::endl;
}
}
return service;
}
@@ -254,7 +229,22 @@ namespace dropshell
// Load environment files
load_env_file(localfile::service_env(server_name, service_name));
load_env_file(localfile::template_info_env(server_name, service_name));
std::string template_name = get_var(all_env_vars, "TEMPLATE");
if (template_name.empty())
{
error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
return false;
}
auto tinfo = gTemplateManager().get_template_info(template_name, true); // skip updates.
if (!tinfo.is_set())
{
// Template is not available locally or from registry
error << "Template '" << template_name << "' not found locally or in registry" << std::endl;
return false;
}
ASSERT(std::filesystem::exists(tinfo.local_template_info_env_path()));
load_env_file(tinfo.local_template_info_env_path());
std::string user = get_var(all_env_vars, "SSH_USER");
if (user.empty())
@@ -269,27 +259,6 @@ namespace dropshell
set_var(all_env_vars, "CONFIG_PATH", remotepath(server_name, user).service_config(service_name));
set_var(all_env_vars, "AGENT_PATH", remotepath(server_name, user).agent());
// determine template name.
auto it = find_var(all_env_vars, "TEMPLATE");
if (it == all_env_vars.end())
{
error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
info << "The TEMPLATE variable is required to determine the template name." << std::endl;
info << "Please check the " << filenames::service_env << " file and the "<< filenames::template_info_env << " file in:" << std::endl;
info << " " << localpath::service(server_name, service_name) << std::endl
<< std::endl;
return false;
}
// Try to get the template - this will download from registry if needed
template_info tinfo = gTemplateManager().get_template_info(it->second);
if (!tinfo.is_set())
{
// Template is not available locally or from registry
error << "Template '" << it->second << "' not found locally or in registry" << std::endl;
return false;
}
return true;
}

8
source/src/services.hpp
View File

@@ -18,15 +18,15 @@ namespace dropshell {
bool requires_host_root;
bool requires_docker;
bool requires_docker_root;
bool service_template_hash_match;
//bool service_template_hash_match;
};
bool SIvalid(const LocalServiceInfo& service_info);
// if fast, don't check for updates to the service template.
std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name, bool fast=false);
// if skip_update, don't check for updates to the service template.
std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name, bool skip_update=false);
LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool fast=false);
LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool skip_update=false);
std::set<std::string> get_used_commands(const std::string& server_name, const std::string& service_name);
// get all env vars for a given service

86
source/src/templates.cpp
View File

@@ -48,7 +48,7 @@
return std::filesystem::exists(path);
}
template_info template_source_local::get_template_info(const std::string& template_name, bool fast) {
template_info template_source_local::get_template_info(const std::string& template_name, bool skip_update) {
std::filesystem::path path = mLocalPath / template_name;
if (!std::filesystem::exists(path))
@@ -58,7 +58,7 @@
template_name,
mLocalPath.string(),
path,
fast
skip_update
);
}
@@ -183,7 +183,7 @@
return std::filesystem::exists(template_cache_dir);
}
template_info template_source_registry::get_template_info(const std::string& template_name, bool fast)
template_info template_source_registry::get_template_info(const std::string& template_name, bool skip_update)
{
// Get cache directory
std::filesystem::path cache_dir = get_cache_dir();
@@ -203,13 +203,13 @@
return template_info();
}
// fast: don't bother updating anything - if we have a cached version use that.
if (fast && have_cache) {
// skip_update: don't bother updating anything - if we have a cached version use that.
if (skip_update && have_cache) {
return template_info(
template_name,
"Registry: " + mRegistry.name + " (cached)",
template_cache_dir,
fast
skip_update
);
}
@@ -248,7 +248,7 @@
template_name,
"Registry: " + mRegistry.name + " (cached)",
template_cache_dir,
fast
skip_update
);
}
warning << "Failed to get metadata for template: " << template_name << std::endl;
@@ -258,7 +258,7 @@
// Check if we need to download/update the template
bool need_download = true;
std::string registry_version = "unknown";
std::string registry_hash = "";
std::string registry_unpacked_hash = "";
// Extract version and hash from registry metadata
if (registry_metadata.contains("metadata")) {
@@ -268,7 +268,7 @@
}
// REQUIRED: unpackedhash - the hash of extracted contents
if (metadata.contains("unpackedhash")) {
registry_hash = metadata["unpackedhash"].get<std::string>();
registry_unpacked_hash = metadata["unpackedhash"].get<std::string>();
//debug << "Found unpackedhash in metadata: " << registry_hash << std::endl;
} else {
// unpackedhash is required for security
@@ -289,17 +289,11 @@
cache_file.close();
// Compare versions or hashes
if (cache_json.contains("hash") && !registry_hash.empty()) {
if (cache_json["hash"].get<std::string>() == registry_hash) {
if (cache_json.contains("unpacked_hash") && !registry_unpacked_hash.empty())
if (cache_json["unpacked_hash"].get<std::string>() == registry_unpacked_hash)
need_download = false;
}
} else if (cache_json.contains("version")) {
std::string cached_version = cache_json["version"].get<std::string>();
if (cached_version == registry_version) {
need_download = false;
}
}
} catch (...) {
catch (...) {
// If reading cache fails, re-download
need_download = true;
}
@@ -336,21 +330,21 @@
std::filesystem::remove(temp_tgz);
// Calculate actual hash of extracted template
std::string actual_hash = hash_directory_recursive(template_cache_dir.string());
std::string actual_unpacked_hash = hash_directory_recursive(template_cache_dir.string());
// Verify the extracted template hash matches what registry claimed
// unpackedhash is required, so registry_hash should always be set here
if (registry_hash.empty()) {
if (registry_unpacked_hash.empty()) {
// This shouldn't happen as we check for it above, but handle it just in case
error << "Internal error: unpackedhash was not properly set" << std::endl;
std::filesystem::remove_all(template_cache_dir);
return template_info();
}
if (actual_hash != registry_hash) {
if (actual_unpacked_hash != registry_unpacked_hash) {
error << "Template hash verification failed!" << std::endl;
error << "Expected hash: " << registry_hash << std::endl;
error << "Actual hash: " << actual_hash << std::endl;
error << "Expected unpacked hash: " << registry_unpacked_hash << std::endl;
error << "Actual unpacked hash: " << actual_unpacked_hash << std::endl;
error << "The downloaded template '" << template_name << "' may be corrupted or tampered with." << std::endl;
// Remove the corrupted template
@@ -358,32 +352,16 @@
return template_info();
}
info << "Template extracted successfully. SHA256: " << actual_hash << std::endl;
info << "Note: Hash verification temporarily disabled during migration from XXHash64 to SHA256" << std::endl;
info << "Template extracted successfully. SHA256: " << actual_unpacked_hash << std::endl;
// Generate .template_info.env if it doesn't exist
std::filesystem::path template_info_env_path = template_cache_dir / "config" / filenames::template_info_env;
if (!std::filesystem::exists(template_info_env_path)) {
// Create config directory if needed
std::filesystem::create_directories(template_cache_dir / "config");
// Write .template_info.env file
std::ofstream info_file(template_info_env_path);
info_file << "# Template information" << std::endl;
info_file << "TEMPLATE=" << template_name << std::endl;
info_file << "TEMPLATE_SOURCE=registry" << std::endl;
info_file << "TEMPLATE_REGISTRY=" << mRegistry.name << std::endl;
info_file << "TEMPLATE_VERSION=" << registry_version << std::endl;
// Always write the actual calculated hash
info_file << "TEMPLATE_HASH=" << actual_hash << std::endl;
info_file.close();
}
std::filesystem::path template_info_env_path = template_cache_dir / filenames::template_info_env;
ASSERT( std::filesystem::exists( template_info_env_path ), "template_info.env doesn't exist in the template." );
// Update cache JSON file
nlohmann::json cache_json;
cache_json["template"] = template_name;
cache_json["version"] = registry_version;
cache_json["hash"] = actual_hash; // Store actual calculated hash
cache_json["unpacked_hash"] = actual_unpacked_hash; // Store actual calculated hash
cache_json["registry"] = mRegistry.name;
cache_json["last_updated"] = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
@@ -399,7 +377,7 @@
template_name,
"Registry: " + mRegistry.name,
template_cache_dir,
fast
skip_update
);
}
@@ -471,12 +449,12 @@
return true;
}
template_info template_manager::get_template_info(const std::string &template_name, bool fast) const
template_info template_manager::get_template_info(const std::string &template_name, bool skip_update) const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
template_source_interface* source = get_source(template_name);
if (source)
return source->get_template_info(template_name, fast);
return source->get_template_info(template_name, skip_update);
// fail
return template_info();
@@ -797,7 +775,7 @@ For full documentation, see: dropshell help templates
std::vector<std::string> required_files = {
"config/" + filenames::service_env,
"config/" + filenames::template_info_env,
filenames::template_info_env,
"install.sh",
"uninstall.sh"
};
@@ -899,13 +877,12 @@ For full documentation, see: dropshell help templates
return instance;
}
template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path, bool fast) :
template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path, bool skip_update) :
mTemplateName(template_name),
mLocationID(location_id),
mTemplateLocalPath(local_template_path),
mTemplateValid(template_manager::test_template(local_template_path.string())),
mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty()),\
mHash("")
mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty())
{
if (!std::filesystem::exists(local_template_path))
{
@@ -913,8 +890,13 @@ For full documentation, see: dropshell help templates
return;
}
if (!fast)
mHash = hash_directory_recursive(local_template_path);
// if (!skip_update)
// mHash = hash_directory_recursive(local_template_path);
}
std::filesystem::path template_info::local_template_info_env_path()
{
return mTemplateLocalPath / filenames::template_info_env ;
}
} // namespace dropshell

13
source/src/templates.hpp
View File

@@ -20,14 +20,14 @@ typedef enum template_source_type {
class template_info {
public:
template_info() : mIsSet(false) {}
template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool fast);
template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool skip_update);
virtual ~template_info() {}
bool is_set() const { return mIsSet; }
std::string name() const { return mTemplateName; }
std::string locationID() const { return mLocationID; }
std::filesystem::path local_template_path() const { return mTemplateLocalPath; }
std::filesystem::path local_template_info_env_path();
bool template_valid() const { return mTemplateValid; }
std::string hash() const { return mHash; }
private:
std::string mTemplateName;
@@ -35,7 +35,6 @@ class template_info {
std::filesystem::path mTemplateLocalPath; // source or cache.
bool mTemplateValid;
bool mIsSet;
std::string mHash;
};
class template_source_interface {
@@ -43,7 +42,7 @@ class template_source_interface {
virtual ~template_source_interface() {}
virtual std::set<std::string> get_template_list() = 0;
virtual bool has_template(const std::string& template_name) = 0;
virtual template_info get_template_info(const std::string& template_name, bool fast=false) = 0;
virtual template_info get_template_info(const std::string& template_name, bool skip_update=false) = 0;
virtual bool template_command_exists(const std::string& template_name,const std::string& command) = 0;
virtual std::string get_description() = 0;
@@ -57,7 +56,7 @@ class template_source_registry : public template_source_interface {
std::set<std::string> get_template_list();
bool has_template(const std::string& template_name);
template_info get_template_info(const std::string& template_name, bool fast=false);
template_info get_template_info(const std::string& template_name, bool skip_update=false);
bool template_command_exists(const std::string& template_name,const std::string& command);
std::string get_description() { return "Registry: " + mRegistry.name + " (" + mRegistry.url + ")"; }
@@ -74,7 +73,7 @@ class template_source_local : public template_source_interface {
~template_source_local() {}
std::set<std::string> get_template_list();
bool has_template(const std::string& template_name);
template_info get_template_info(const std::string& template_name, bool fast=false);
template_info get_template_info(const std::string& template_name, bool skip_update=false);
bool template_command_exists(const std::string& template_name,const std::string& command);
std::string get_description() { return "Local: " + mLocalPath.string(); }
@@ -89,7 +88,7 @@ class template_manager {
std::set<std::string> get_template_list() const;
bool has_template(const std::string& template_name) const;
template_info get_template_info(const std::string& template_name, bool fast=false) const; // fast = don't check for updates.
template_info get_template_info(const std::string& template_name, bool skip_update=false) const; // skip_update = don't check for updates.
bool template_command_exists(const std::string& template_name,const std::string& command) const;
bool create_template(const std::string& template_name) const;

11
source/src/utils/directories.cpp
View File

@@ -32,11 +32,6 @@ namespace dropshell
return (servicepath.empty() ? "" : (fs::path(servicepath) / filenames::service_env).string());
}
std::string template_example()
{
return localpath::agent_local() + "/template_example";
}
std::string bb64()
{
return localpath::agent_local() + "/bb64";
@@ -114,6 +109,12 @@ namespace dropshell
return dropshell_dir() + "/template_cache";
}
std::string template_example()
{
return agent_local() + "/template_example";
}
bool create_directories()
{
std::vector<std::filesystem::path> paths = {

3
source/src/utils/directories.hpp
View File

@@ -53,7 +53,6 @@ namespace dropshell {
std::string dropshell_json();
std::string server_json(const std::string &server_name);
std::string service_env(const std::string &server_name, const std::string &service_name);
std::string template_example();
std::string bb64();
} // namespace localfile
@@ -68,6 +67,8 @@ namespace dropshell {
std::string current_user_home();
std::string user_local_bin(); // ~/.local/bin directory for user executables
std::string template_example();
std::string backups();
std::string temp_files();
std::string template_cache();