diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index a07d8be..6a3c0cb 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -4,7 +4,9 @@ "name": "Linux", "includePath": [ "${workspaceFolder}/**", - "${workspaceFolder}/source/build/src/autogen" + "${workspaceFolder}/source/build/src/autogen", + "${workspaceFolder}/build/_deps/libassert-src/include", + "${workspaceFolder}/build/_deps/cpptrace-src/include" ], "defines": [], "compilerPath": "/usr/bin/g++", diff --git a/source/src/commands/create-service.cpp b/source/src/commands/create-service.cpp index 4e8e254..5f5407b 100644 --- a/source/src/commands/create-service.cpp +++ b/source/src/commands/create-service.cpp @@ -156,13 +156,13 @@ namespace dropshell // copy the template config files to the service directory recursive_copy(tinfo.local_template_path() / "config", service_dir); - // append TEMPLATE_HASH to the .template_info.env file - std::string template_info_env_file = localfile::template_info_env(server_name,service_name); - ASSERT(std::filesystem::exists(template_info_env_file), "Template info env file not found: " + template_info_env_file); - std::ofstream template_info_env_file_out(template_info_env_file, std::ios::app); // append to the file. - template_info_env_file_out << std::endl << "TEMPLATE_HASH=" << tinfo.hash() << std::endl; - template_info_env_file_out.close(); - + // append TEMPLATE to the service.env file + std::string service_env_file = localfile::service_env(server_name, service_name); + ASSERT(std::filesystem::exists(service_env_file),"service.env not found: "+ service_env_file); + std::ofstream ofs(service_env_file, std::ios::app); + ofs << std::endl << "# TEMPLATE set by dropshell on creation. Important this does not change." << std::endl; + ofs << "TEMPLATE=\"" << template_name <<"\"" << std::endl; + ofs.close(); // modify the SSH_USER to be nice. // everything is created, so we can get the service info. @@ -245,34 +245,6 @@ namespace dropshell return true; } - - bool merge_updated_service_template(const std::string &server_name, const std::string &service_name) - { - LocalServiceInfo service_info = get_service_info(server_name, service_name); - ASSERT(SIvalid(service_info), "Service info is not valid for " + service_name + " on " + server_name); - - template_info tinfo = gTemplateManager().get_template_info(service_info.template_name); - ASSERT(tinfo.is_set(), "Failed to load template " + service_info.template_name); - - // copy across .template_info.env file - std::string template_info_env_file = tinfo.local_template_path() / "config" / filenames::template_info_env; - std::string target_template_info_env_file = localfile::template_info_env(server_name, service_name); - ASSERT(std::filesystem::exists(template_info_env_file), "Template service env file not found: " + template_info_env_file); - std::filesystem::remove(target_template_info_env_file); - std::filesystem::copy(template_info_env_file, target_template_info_env_file); - - #pragma message("TODO: merge the template info env file") - - // update hash in template info env file - // append TEMPLATE_HASH to the .template_info.env file - ASSERT(std::filesystem::exists(target_template_info_env_file), "Template info env file not found: " + target_template_info_env_file); - std::ofstream template_info_env_file_out(target_template_info_env_file, std::ios::app); // append to the file. - template_info_env_file_out << std::endl << "TEMPLATE_HASH=" << tinfo.hash() << std::endl; - template_info_env_file_out.close(); - - return true; - } - } // namespace shared_commands } // namespace dropshell \ No newline at end of file diff --git a/source/src/commands/install.cpp b/source/src/commands/install.cpp index 326891d..17f4d24 100644 --- a/source/src/commands/install.cpp +++ b/source/src/commands/install.cpp @@ -93,23 +93,6 @@ namespace dropshell shared_commands::uninstall_service(server_env, service); } - if (!service_info.service_template_hash_match) - { - warning << "Service " << service << " is using an old template. Updating. " << std::endl; - if (!merge_updated_service_template(server_env.get_server_name(), service)) - { - error << "Failed to merge updated service template. " << std::endl; - return false; - } - - service_info = get_service_info(server_env.get_server_name(), service); - if (!SIvalid(service_info) || !service_info.service_template_hash_match) - { - error << "Merged updated service template, but it is still not valid. " << std::endl; - return false; - } - } - maketitle("Installing " + service + " (" + service_info.template_name + ") on " + server); // Check if template exists @@ -541,12 +524,11 @@ complete -F _dropshell_completions ds // copy server.json across. info << "Copying server.json to remote server... " < get_server_services_info(const std::string &server_name, bool fast) + std::vector get_server_services_info(const std::string &server_name, bool skip_update) { std::vector services; @@ -50,7 +49,7 @@ namespace dropshell std::string dirname = entry.path().filename().string(); if (dirname.empty() || dirname[0] == '.' || dirname[0] == '_') continue; - auto service = get_service_info(server_name, dirname, fast); + auto service = get_service_info(server_name, dirname, skip_update); if (!service.local_service_path.empty()) services.push_back(service); else @@ -73,7 +72,7 @@ namespace dropshell return it->second == "true"; } - LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool fast) + LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name, bool skip_update) { LocalServiceInfo service; @@ -111,7 +110,7 @@ namespace dropshell service.template_name = it->second; } - template_info tinfo = gTemplateManager().get_template_info(service.template_name,fast); + template_info tinfo = gTemplateManager().get_template_info(service.template_name,skip_update); if (!tinfo.is_set()) { // Template not found - this means it's not available locally OR from registry @@ -137,30 +136,6 @@ namespace dropshell service.requires_docker = get_bool_variable(variables, "REQUIRES_DOCKER"); service.requires_docker_root = get_bool_variable(variables, "REQUIRES_DOCKER_ROOT"); - { // determine if the service template hash matches the template hash. - auto it = find_var(variables, "TEMPLATE_HASH"); - service.service_template_hash_match = false; - - if (it == variables.end()) - { - // For backward compatibility with services created before hash mechanism - debug << "TEMPLATE_HASH not found in " << filenames::template_info_env << " for " << server_name << " - " << service.template_name - << " (service may have been created before hash tracking was implemented)" << std::endl; - } - else if (tinfo.is_set()) - { - std::string template_hash_str = it->second; - // Compare hash strings directly (migration from uint64_t to SHA256 string) - service.service_template_hash_match = (template_hash_str == tinfo.hash()); - //debug << "Service template hash: " << service_template_hash << " == " << tinfo.hash() << std::endl; - } - else - { - // Template not available yet, can't check hash - debug << "Couldn't check template hash as the template info is not available (yet?)" << std::endl; - } - } - return service; } @@ -254,7 +229,22 @@ namespace dropshell // Load environment files load_env_file(localfile::service_env(server_name, service_name)); - load_env_file(localfile::template_info_env(server_name, service_name)); + + std::string template_name = get_var(all_env_vars, "TEMPLATE"); + if (template_name.empty()) + { + error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl; + return false; + } + auto tinfo = gTemplateManager().get_template_info(template_name, true); // skip updates. + if (!tinfo.is_set()) + { + // Template is not available locally or from registry + error << "Template '" << template_name << "' not found locally or in registry" << std::endl; + return false; + } + ASSERT(std::filesystem::exists(tinfo.local_template_info_env_path())); + load_env_file(tinfo.local_template_info_env_path()); std::string user = get_var(all_env_vars, "SSH_USER"); if (user.empty()) @@ -269,27 +259,6 @@ namespace dropshell set_var(all_env_vars, "CONFIG_PATH", remotepath(server_name, user).service_config(service_name)); set_var(all_env_vars, "AGENT_PATH", remotepath(server_name, user).agent()); - - // determine template name. - auto it = find_var(all_env_vars, "TEMPLATE"); - if (it == all_env_vars.end()) - { - error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl; - info << "The TEMPLATE variable is required to determine the template name." << std::endl; - info << "Please check the " << filenames::service_env << " file and the "<< filenames::template_info_env << " file in:" << std::endl; - info << " " << localpath::service(server_name, service_name) << std::endl - << std::endl; - return false; - } - // Try to get the template - this will download from registry if needed - template_info tinfo = gTemplateManager().get_template_info(it->second); - if (!tinfo.is_set()) - { - // Template is not available locally or from registry - error << "Template '" << it->second << "' not found locally or in registry" << std::endl; - return false; - } - return true; } diff --git a/source/src/services.hpp b/source/src/services.hpp index cb60c83..f14c817 100644 --- a/source/src/services.hpp +++ b/source/src/services.hpp @@ -18,15 +18,15 @@ namespace dropshell { bool requires_host_root; bool requires_docker; bool requires_docker_root; - bool service_template_hash_match; + //bool service_template_hash_match; }; bool SIvalid(const LocalServiceInfo& service_info); - // if fast, don't check for updates to the service template. - std::vector get_server_services_info(const std::string& server_name, bool fast=false); + // if skip_update, don't check for updates to the service template. + std::vector get_server_services_info(const std::string& server_name, bool skip_update=false); - LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool fast=false); + LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name, bool skip_update=false); std::set get_used_commands(const std::string& server_name, const std::string& service_name); // get all env vars for a given service diff --git a/source/src/templates.cpp b/source/src/templates.cpp index 2655181..00710e0 100644 --- a/source/src/templates.cpp +++ b/source/src/templates.cpp @@ -48,7 +48,7 @@ return std::filesystem::exists(path); } - template_info template_source_local::get_template_info(const std::string& template_name, bool fast) { + template_info template_source_local::get_template_info(const std::string& template_name, bool skip_update) { std::filesystem::path path = mLocalPath / template_name; if (!std::filesystem::exists(path)) @@ -58,7 +58,7 @@ template_name, mLocalPath.string(), path, - fast + skip_update ); } @@ -183,7 +183,7 @@ return std::filesystem::exists(template_cache_dir); } - template_info template_source_registry::get_template_info(const std::string& template_name, bool fast) + template_info template_source_registry::get_template_info(const std::string& template_name, bool skip_update) { // Get cache directory std::filesystem::path cache_dir = get_cache_dir(); @@ -203,13 +203,13 @@ return template_info(); } - // fast: don't bother updating anything - if we have a cached version use that. - if (fast && have_cache) { + // skip_update: don't bother updating anything - if we have a cached version use that. + if (skip_update && have_cache) { return template_info( template_name, "Registry: " + mRegistry.name + " (cached)", template_cache_dir, - fast + skip_update ); } @@ -248,7 +248,7 @@ template_name, "Registry: " + mRegistry.name + " (cached)", template_cache_dir, - fast + skip_update ); } warning << "Failed to get metadata for template: " << template_name << std::endl; @@ -258,7 +258,7 @@ // Check if we need to download/update the template bool need_download = true; std::string registry_version = "unknown"; - std::string registry_hash = ""; + std::string registry_unpacked_hash = ""; // Extract version and hash from registry metadata if (registry_metadata.contains("metadata")) { @@ -268,7 +268,7 @@ } // REQUIRED: unpackedhash - the hash of extracted contents if (metadata.contains("unpackedhash")) { - registry_hash = metadata["unpackedhash"].get(); + registry_unpacked_hash = metadata["unpackedhash"].get(); //debug << "Found unpackedhash in metadata: " << registry_hash << std::endl; } else { // unpackedhash is required for security @@ -289,17 +289,11 @@ cache_file.close(); // Compare versions or hashes - if (cache_json.contains("hash") && !registry_hash.empty()) { - if (cache_json["hash"].get() == registry_hash) { + if (cache_json.contains("unpacked_hash") && !registry_unpacked_hash.empty()) + if (cache_json["unpacked_hash"].get() == registry_unpacked_hash) need_download = false; - } - } else if (cache_json.contains("version")) { - std::string cached_version = cache_json["version"].get(); - if (cached_version == registry_version) { - need_download = false; - } } - } catch (...) { + catch (...) { // If reading cache fails, re-download need_download = true; } @@ -336,21 +330,21 @@ std::filesystem::remove(temp_tgz); // Calculate actual hash of extracted template - std::string actual_hash = hash_directory_recursive(template_cache_dir.string()); + std::string actual_unpacked_hash = hash_directory_recursive(template_cache_dir.string()); // Verify the extracted template hash matches what registry claimed // unpackedhash is required, so registry_hash should always be set here - if (registry_hash.empty()) { + if (registry_unpacked_hash.empty()) { // This shouldn't happen as we check for it above, but handle it just in case error << "Internal error: unpackedhash was not properly set" << std::endl; std::filesystem::remove_all(template_cache_dir); return template_info(); } - if (actual_hash != registry_hash) { + if (actual_unpacked_hash != registry_unpacked_hash) { error << "Template hash verification failed!" << std::endl; - error << "Expected hash: " << registry_hash << std::endl; - error << "Actual hash: " << actual_hash << std::endl; + error << "Expected unpacked hash: " << registry_unpacked_hash << std::endl; + error << "Actual unpacked hash: " << actual_unpacked_hash << std::endl; error << "The downloaded template '" << template_name << "' may be corrupted or tampered with." << std::endl; // Remove the corrupted template @@ -358,32 +352,16 @@ return template_info(); } - info << "Template extracted successfully. SHA256: " << actual_hash << std::endl; - info << "Note: Hash verification temporarily disabled during migration from XXHash64 to SHA256" << std::endl; + info << "Template extracted successfully. SHA256: " << actual_unpacked_hash << std::endl; - // Generate .template_info.env if it doesn't exist - std::filesystem::path template_info_env_path = template_cache_dir / "config" / filenames::template_info_env; - if (!std::filesystem::exists(template_info_env_path)) { - // Create config directory if needed - std::filesystem::create_directories(template_cache_dir / "config"); - - // Write .template_info.env file - std::ofstream info_file(template_info_env_path); - info_file << "# Template information" << std::endl; - info_file << "TEMPLATE=" << template_name << std::endl; - info_file << "TEMPLATE_SOURCE=registry" << std::endl; - info_file << "TEMPLATE_REGISTRY=" << mRegistry.name << std::endl; - info_file << "TEMPLATE_VERSION=" << registry_version << std::endl; - // Always write the actual calculated hash - info_file << "TEMPLATE_HASH=" << actual_hash << std::endl; - info_file.close(); - } + std::filesystem::path template_info_env_path = template_cache_dir / filenames::template_info_env; + ASSERT( std::filesystem::exists( template_info_env_path ), "template_info.env doesn't exist in the template." ); // Update cache JSON file nlohmann::json cache_json; cache_json["template"] = template_name; cache_json["version"] = registry_version; - cache_json["hash"] = actual_hash; // Store actual calculated hash + cache_json["unpacked_hash"] = actual_unpacked_hash; // Store actual calculated hash cache_json["registry"] = mRegistry.name; cache_json["last_updated"] = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); @@ -399,7 +377,7 @@ template_name, "Registry: " + mRegistry.name, template_cache_dir, - fast + skip_update ); } @@ -471,12 +449,12 @@ return true; } - template_info template_manager::get_template_info(const std::string &template_name, bool fast) const + template_info template_manager::get_template_info(const std::string &template_name, bool skip_update) const { ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found."); template_source_interface* source = get_source(template_name); if (source) - return source->get_template_info(template_name, fast); + return source->get_template_info(template_name, skip_update); // fail return template_info(); @@ -797,7 +775,7 @@ For full documentation, see: dropshell help templates std::vector required_files = { "config/" + filenames::service_env, - "config/" + filenames::template_info_env, + filenames::template_info_env, "install.sh", "uninstall.sh" }; @@ -899,13 +877,12 @@ For full documentation, see: dropshell help templates return instance; } - template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path, bool fast) : + template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path, bool skip_update) : mTemplateName(template_name), mLocationID(location_id), mTemplateLocalPath(local_template_path), mTemplateValid(template_manager::test_template(local_template_path.string())), - mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty()),\ - mHash("") + mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty()) { if (!std::filesystem::exists(local_template_path)) { @@ -913,8 +890,13 @@ For full documentation, see: dropshell help templates return; } - if (!fast) - mHash = hash_directory_recursive(local_template_path); + // if (!skip_update) + // mHash = hash_directory_recursive(local_template_path); + } + + std::filesystem::path template_info::local_template_info_env_path() + { + return mTemplateLocalPath / filenames::template_info_env ; } } // namespace dropshell diff --git a/source/src/templates.hpp b/source/src/templates.hpp index 8cd1c4e..f0077cc 100644 --- a/source/src/templates.hpp +++ b/source/src/templates.hpp @@ -20,14 +20,14 @@ typedef enum template_source_type { class template_info { public: template_info() : mIsSet(false) {} - template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool fast); + template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path, bool skip_update); virtual ~template_info() {} bool is_set() const { return mIsSet; } std::string name() const { return mTemplateName; } std::string locationID() const { return mLocationID; } std::filesystem::path local_template_path() const { return mTemplateLocalPath; } + std::filesystem::path local_template_info_env_path(); bool template_valid() const { return mTemplateValid; } - std::string hash() const { return mHash; } private: std::string mTemplateName; @@ -35,7 +35,6 @@ class template_info { std::filesystem::path mTemplateLocalPath; // source or cache. bool mTemplateValid; bool mIsSet; - std::string mHash; }; class template_source_interface { @@ -43,7 +42,7 @@ class template_source_interface { virtual ~template_source_interface() {} virtual std::set get_template_list() = 0; virtual bool has_template(const std::string& template_name) = 0; - virtual template_info get_template_info(const std::string& template_name, bool fast=false) = 0; + virtual template_info get_template_info(const std::string& template_name, bool skip_update=false) = 0; virtual bool template_command_exists(const std::string& template_name,const std::string& command) = 0; virtual std::string get_description() = 0; @@ -57,7 +56,7 @@ class template_source_registry : public template_source_interface { std::set get_template_list(); bool has_template(const std::string& template_name); - template_info get_template_info(const std::string& template_name, bool fast=false); + template_info get_template_info(const std::string& template_name, bool skip_update=false); bool template_command_exists(const std::string& template_name,const std::string& command); std::string get_description() { return "Registry: " + mRegistry.name + " (" + mRegistry.url + ")"; } @@ -74,7 +73,7 @@ class template_source_local : public template_source_interface { ~template_source_local() {} std::set get_template_list(); bool has_template(const std::string& template_name); - template_info get_template_info(const std::string& template_name, bool fast=false); + template_info get_template_info(const std::string& template_name, bool skip_update=false); bool template_command_exists(const std::string& template_name,const std::string& command); std::string get_description() { return "Local: " + mLocalPath.string(); } @@ -89,7 +88,7 @@ class template_manager { std::set get_template_list() const; bool has_template(const std::string& template_name) const; - template_info get_template_info(const std::string& template_name, bool fast=false) const; // fast = don't check for updates. + template_info get_template_info(const std::string& template_name, bool skip_update=false) const; // skip_update = don't check for updates. bool template_command_exists(const std::string& template_name,const std::string& command) const; bool create_template(const std::string& template_name) const; diff --git a/source/src/utils/directories.cpp b/source/src/utils/directories.cpp index 59f8f7d..d82e3be 100644 --- a/source/src/utils/directories.cpp +++ b/source/src/utils/directories.cpp @@ -32,11 +32,6 @@ namespace dropshell return (servicepath.empty() ? "" : (fs::path(servicepath) / filenames::service_env).string()); } - std::string template_example() - { - return localpath::agent_local() + "/template_example"; - } - std::string bb64() { return localpath::agent_local() + "/bb64"; @@ -114,6 +109,12 @@ namespace dropshell return dropshell_dir() + "/template_cache"; } + std::string template_example() + { + return agent_local() + "/template_example"; + } + + bool create_directories() { std::vector paths = { diff --git a/source/src/utils/directories.hpp b/source/src/utils/directories.hpp index e3d5be1..7ef52fd 100644 --- a/source/src/utils/directories.hpp +++ b/source/src/utils/directories.hpp @@ -53,7 +53,6 @@ namespace dropshell { std::string dropshell_json(); std::string server_json(const std::string &server_name); std::string service_env(const std::string &server_name, const std::string &service_name); - std::string template_example(); std::string bb64(); } // namespace localfile @@ -68,6 +67,8 @@ namespace dropshell { std::string current_user_home(); std::string user_local_bin(); // ~/.local/bin directory for user executables + std::string template_example(); + std::string backups(); std::string temp_files(); std::string template_cache();