From af75b0b4ac4c828a116ebdfd967a5ac3c7549516 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 3 Oct 2025 18:56:04 +1300 Subject: [PATCH] fix env variable load and subst order --- source/src/commands/create-service.cpp | 6 +-- source/src/commands/exec.cpp | 6 +-- source/src/servers.cpp | 4 +- source/src/services.cpp | 34 +++++++------- source/src/services.hpp | 3 +- source/src/templates.cpp | 8 ++-- source/src/utils/envmanager.cpp | 20 +++----- source/src/utils/envmanager.hpp | 8 ++-- source/src/utils/execute.cpp | 2 +- source/src/utils/execute.hpp | 15 +++--- source/src/utils/ordered_env.hpp | 64 ++++++++++++++++++++++++++ source/src/utils/utils.cpp | 15 +++--- source/src/utils/utils.hpp | 3 +- 13 files changed, 124 insertions(+), 64 deletions(-) create mode 100644 source/src/utils/ordered_env.hpp diff --git a/source/src/commands/create-service.cpp b/source/src/commands/create-service.cpp index ef44af7..4e8e254 100644 --- a/source/src/commands/create-service.cpp +++ b/source/src/commands/create-service.cpp @@ -83,10 +83,10 @@ namespace dropshell if (!std::filesystem::exists(readme_path)) return false; - std::map all_env_vars; + ordered_env_vars all_env_vars; get_all_service_env_vars(server, service, all_env_vars); - all_env_vars["LOCAL_CONFIG_PATH"] = localpath::service(server, service); - all_env_vars["LOCAL_TEMPLATE_PATH"] = tinfo.local_template_path().string(); + set_var(all_env_vars, "LOCAL_CONFIG_PATH", localpath::service(server, service)); + set_var(all_env_vars, "LOCAL_TEMPLATE_PATH", tinfo.local_template_path().string()); info << std::endl; std::ifstream readme_file(readme_path); diff --git a/source/src/commands/exec.cpp b/source/src/commands/exec.cpp index a6552d3..c97724c 100644 --- a/source/src/commands/exec.cpp +++ b/source/src/commands/exec.cpp @@ -89,15 +89,15 @@ namespace dropshell std::string user = server_env.get_user_for_service(service); // Get all service environment variables - std::map env_vars; + ordered_env_vars env_vars; if (!get_all_service_env_vars(server, service, env_vars)) { error << "Failed to get environment variables for service " << service << std::endl; return 1; } - + // Add HOST_NAME like other commands do - env_vars["HOST_NAME"] = server_env.get_SSH_HOST(); + set_var(env_vars, "HOST_NAME", server_env.get_SSH_HOST()); // Get the remote service template path for working directory std::string remote_service_template_path = remotepath(server, user).service_template(service); diff --git a/source/src/servers.cpp b/source/src/servers.cpp index cb02566..ab2fe64 100644 --- a/source/src/servers.cpp +++ b/source/src/servers.cpp @@ -347,14 +347,14 @@ namespace dropshell std::string remote_service_template_path = remotepath(mServerName, user).service_template(service_name); std::string script_path = remote_service_template_path + "/" + command + ".sh"; - std::map env_vars; + ordered_env_vars env_vars; if (!get_all_service_env_vars(mServerName, service_name, env_vars)) { error << "Failed to get all service env vars for " << service_name << std::endl; return std::nullopt; } - env_vars["HOST_NAME"] = get_SSH_HOST(); + set_var(env_vars, "HOST_NAME", get_SSH_HOST()); std::string argstr = ""; for (const auto &arg : args) diff --git a/source/src/services.cpp b/source/src/services.cpp index dd5be23..af0c132 100644 --- a/source/src/services.cpp +++ b/source/src/services.cpp @@ -62,9 +62,9 @@ namespace dropshell return services; } - bool get_bool_variable(const std::map &variables, const std::string &variable_name) + bool get_bool_variable(const ordered_env_vars &variables, const std::string &variable_name) { - auto it = variables.find(variable_name); + auto it = find_var(variables, variable_name); if (it == variables.end()) { error << "Variable " << variable_name << " not found in the service " << filenames::template_info_env << std::endl; @@ -97,12 +97,12 @@ namespace dropshell } // now set the template name and path. - std::map variables; + ordered_env_vars variables; if (!get_all_service_env_vars(server_name, service_name, variables)) return LocalServiceInfo(); { // confirm TEMPLATE is defined. - auto it = variables.find("TEMPLATE"); + auto it = find_var(variables, "TEMPLATE"); if (it == variables.end()) { error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl; @@ -123,7 +123,7 @@ namespace dropshell service.local_template_path = tinfo.local_template_path(); { // set the user. - auto it = variables.find("SSH_USER"); + auto it = find_var(variables, "SSH_USER"); if (it == variables.end()) { error << "SSH_USER variable not defined in service " << service_name << " on server " << server_name << std::endl; @@ -138,7 +138,7 @@ namespace dropshell service.requires_docker_root = get_bool_variable(variables, "REQUIRES_DOCKER_ROOT"); { // determine if the service template hash matches the template hash. - auto it = variables.find("TEMPLATE_HASH"); + auto it = find_var(variables, "TEMPLATE_HASH"); service.service_template_hash_match = false; if (it == variables.end()) @@ -221,9 +221,9 @@ namespace dropshell return backups; } - bool get_all_service_env_vars(const std::string &server_name, const std::string &service_name, std::map &all_env_vars) + bool get_all_service_env_vars(const std::string &server_name, const std::string &service_name, ordered_env_vars &all_env_vars) { - all_env_vars.clear(); + clear_vars(all_env_vars); if (localpath::service(server_name, service_name).empty() || !fs::exists(localpath::service(server_name, service_name))) { @@ -236,11 +236,11 @@ namespace dropshell { if (!file.empty() && std::filesystem::exists(file)) { - std::map env_vars; + ordered_env_vars env_vars; envmanager env_manager(file); env_manager.load(); env_manager.get_all_variables(env_vars); - all_env_vars.merge(env_vars); + merge_vars(all_env_vars, env_vars); } else warning << "Expected environment file not found: " << file << std::endl; @@ -248,15 +248,15 @@ namespace dropshell // add in some simple variables first, as others below may depend on/use these in bash. - all_env_vars["SERVER"] = server_name; - all_env_vars["SERVICE"] = service_name; - all_env_vars["DOCKER_CLI_HINTS"] = "false"; // turn off docker junk. + set_var(all_env_vars, "SERVER", server_name); + set_var(all_env_vars, "SERVICE", service_name); + set_var(all_env_vars, "DOCKER_CLI_HINTS", "false"); // turn off docker junk. // Load environment files load_env_file(localfile::service_env(server_name, service_name)); load_env_file(localfile::template_info_env(server_name, service_name)); - std::string user = all_env_vars["SSH_USER"]; + std::string user = get_var(all_env_vars, "SSH_USER"); if (user.empty()) { error << "SSH_USER variable not defined in service " << service_name << " on server " << server_name << std::endl; @@ -266,12 +266,12 @@ namespace dropshell } // more additional, these depend on others above. - all_env_vars["CONFIG_PATH"] = remotepath(server_name, user).service_config(service_name); - all_env_vars["AGENT_PATH"] = remotepath(server_name, user).agent(); + set_var(all_env_vars, "CONFIG_PATH", remotepath(server_name, user).service_config(service_name)); + set_var(all_env_vars, "AGENT_PATH", remotepath(server_name, user).agent()); // determine template name. - auto it = all_env_vars.find("TEMPLATE"); + auto it = find_var(all_env_vars, "TEMPLATE"); if (it == all_env_vars.end()) { error << "TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl; diff --git a/source/src/services.hpp b/source/src/services.hpp index 5964f3d..40caf0b 100644 --- a/source/src/services.hpp +++ b/source/src/services.hpp @@ -5,6 +5,7 @@ #include #include #include +#include "utils/ordered_env.hpp" namespace dropshell { @@ -29,7 +30,7 @@ namespace dropshell { std::set get_used_commands(const std::string& server_name, const std::string& service_name); // get all env vars for a given service - bool get_all_service_env_vars(const std::string& server_name, const std::string& service_name, std::map & all_env_vars); + bool get_all_service_env_vars(const std::string& server_name, const std::string& service_name, ordered_env_vars& all_env_vars); // list all backups for a given service (across all servers) std::set list_backups(const std::string& server_name, const std::string& service_name); diff --git a/source/src/templates.cpp b/source/src/templates.cpp index 46c9f35..12553ad 100644 --- a/source/src/templates.cpp +++ b/source/src/templates.cpp @@ -665,23 +665,23 @@ // ------------------------------------------------------------ // check TEMPLATE= line. - std::map all_env_vars; + ordered_env_vars all_env_vars; std::vector env_files = { "config/" + filenames::service_env, "config/" + filenames::template_info_env }; for (const auto& file : env_files) { { // load service.env from the service on this machine. - std::map env_vars; + ordered_env_vars env_vars; envmanager env_manager(template_path + "/" + file); env_manager.load(); env_manager.get_all_variables(env_vars); - all_env_vars.merge(env_vars); + merge_vars(all_env_vars, env_vars); } } // determine template name. - auto it = all_env_vars.find("TEMPLATE"); + auto it = find_var(all_env_vars, "TEMPLATE"); if (it == all_env_vars.end()) { error << "TEMPLATE variable not found in " << template_path << std::endl; return false; diff --git a/source/src/utils/envmanager.cpp b/source/src/utils/envmanager.cpp index 13ea373..2de3433 100644 --- a/source/src/utils/envmanager.cpp +++ b/source/src/utils/envmanager.cpp @@ -56,7 +56,7 @@ bool envmanager::load() { } // trim whitespace from the key and value - m_variables[dequote(trim(key))] = dequote(trim(value)); + set_var(m_variables, dequote(trim(key)), dequote(trim(value))); } } file.close(); @@ -77,29 +77,21 @@ void envmanager::save() { std::string envmanager::get_variable(std::string key) const { key = dequote(trim(key)); - - // Use case-insensitive comparison to find the key - for (const auto& pair : m_variables) { - if (pair.first == key) { - return pair.second; - } - } - - return ""; + return get_var(m_variables, key); } -void envmanager::get_all_variables(std::map& variables) const { +void envmanager::get_all_variables(ordered_env_vars& variables) const { variables = m_variables; } -void envmanager::add_variables(std::map variables) { - for (auto& pair : variables) { +void envmanager::add_variables(const ordered_env_vars& variables) { + for (const auto& pair : variables) { set_variable(pair.first, pair.second); } } void envmanager::set_variable(std::string key, std::string value) { - m_variables[dequote(trim(key))] = dequote(trim(value)); + set_var(m_variables, dequote(trim(key)), dequote(trim(value))); } void envmanager::clear_variables() { diff --git a/source/src/utils/envmanager.hpp b/source/src/utils/envmanager.hpp index 45236f2..1c54bf8 100644 --- a/source/src/utils/envmanager.hpp +++ b/source/src/utils/envmanager.hpp @@ -4,6 +4,8 @@ #include #include #include +#include "ordered_env.hpp" + namespace dropshell { // envmanager is a class that manages the environment files for the application. @@ -23,17 +25,17 @@ class envmanager { // get variables from the environment files. Trim whitespace from the values. // keys are case-sensitive. std::string get_variable(std::string key) const; - void get_all_variables(std::map& variables) const; + void get_all_variables(ordered_env_vars& variables) const; // add variables to the environment files. // trim whitespace from the values. - void add_variables(std::map variables); + void add_variables(const ordered_env_vars& variables); void set_variable(std::string key, std::string value); void clear_variables(); private: std::string m_path; - std::map m_variables; + ordered_env_vars m_variables; }; // utility functions diff --git a/source/src/utils/execute.cpp b/source/src/utils/execute.cpp index 8cedfc1..893d333 100644 --- a/source/src/utils/execute.cpp +++ b/source/src/utils/execute.cpp @@ -113,7 +113,7 @@ namespace dropshell sColour currentColour_; }; - bool execute_local_command(std::string directory_to_run_in, std::string command_to_run, const std::map &env_vars, std::string *output, cMode mode) + bool execute_local_command(std::string directory_to_run_in, std::string command_to_run, const ordered_env_vars& env_vars, std::string *output, cMode mode) { sCommand command(directory_to_run_in, command_to_run, env_vars); diff --git a/source/src/utils/execute.hpp b/source/src/utils/execute.hpp index ca71734..18297ff 100644 --- a/source/src/utils/execute.hpp +++ b/source/src/utils/execute.hpp @@ -3,6 +3,7 @@ #include #include +#include "ordered_env.hpp" namespace dropshell { @@ -44,7 +45,7 @@ class sSSHInfo { std::string user_dir; // dropshell directory for the user. }; -bool execute_local_command(std::string directory_to_run_in, std::string command_to_run, const std::map & env_vars, std::string * output = nullptr, cMode mode = cMode::Defaults); +bool execute_local_command(std::string directory_to_run_in, std::string command_to_run, const ordered_env_vars& env_vars, std::string * output = nullptr, cMode mode = cMode::Defaults); bool execute_ssh_command(const sSSHInfo & ssh_info, const sCommand & remote_command, cMode mode = cMode::Defaults, std::string * output = nullptr); @@ -54,17 +55,17 @@ bool execute_ssh_command(const sSSHInfo & ssh_info, const sCommand & remote_comm // class to hold a command to run on the remote server. class sCommand { public: - sCommand(std::string directory_to_run_in, std::string command_to_run, const std::map & env_vars) : + sCommand(std::string directory_to_run_in, std::string command_to_run, const ordered_env_vars& env_vars) : mDir(directory_to_run_in), mCmd(command_to_run), mVars(env_vars) {} std::string get_directory_to_run_in() const { return mDir; } std::string get_command_to_run() const { return mCmd; } - const std::map& get_env_vars() const { return mVars; } + const ordered_env_vars& get_env_vars() const { return mVars; } + + void add_env_var(const std::string& key, const std::string& value) { mVars.emplace_back(key, value); } - void add_env_var(const std::string& key, const std::string& value) { mVars[key] = value; } - bool empty() const { return mCmd.empty(); } - + std::string construct_cmd(std::string bb64path) const; private: @@ -73,7 +74,7 @@ class sCommand { private: std::string mDir; std::string mCmd; - std::map mVars; + ordered_env_vars mVars; }; bool EXITSTATUSCHECK(int ret); diff --git a/source/src/utils/ordered_env.hpp b/source/src/utils/ordered_env.hpp new file mode 100644 index 0000000..145ff3d --- /dev/null +++ b/source/src/utils/ordered_env.hpp @@ -0,0 +1,64 @@ +#ifndef ORDERED_ENV_HPP +#define ORDERED_ENV_HPP + +#include +#include +#include +#include + +namespace dropshell { + +// Type alias for insertion-ordered environment variables +using ordered_env_vars = std::vector>; + +// Helper functions for working with ordered_env_vars like a map + +// Find a variable by key (returns iterator) +inline auto find_var(ordered_env_vars& vars, const std::string& key) { + return std::find_if(vars.begin(), vars.end(), + [&](const auto& p) { return p.first == key; }); +} + +inline auto find_var(const ordered_env_vars& vars, const std::string& key) { + return std::find_if(vars.begin(), vars.end(), + [&](const auto& p) { return p.first == key; }); +} + +// Get a variable value (returns empty string if not found) +inline std::string get_var(const ordered_env_vars& vars, const std::string& key) { + auto it = find_var(vars, key); + return (it != vars.end()) ? it->second : ""; +} + +// Set a variable (updates if exists, appends if new) +inline void set_var(ordered_env_vars& vars, const std::string& key, const std::string& value) { + auto it = find_var(vars, key); + if (it != vars.end()) { + it->second = value; + } else { + vars.emplace_back(key, value); + } +} + +// Check if a variable exists +inline bool has_var(const ordered_env_vars& vars, const std::string& key) { + return find_var(vars, key) != vars.end(); +} + +// Merge variables from another ordered_env_vars (appends new, skips existing) +inline void merge_vars(ordered_env_vars& dest, const ordered_env_vars& src) { + for (const auto& [key, value] : src) { + if (!has_var(dest, key)) { + dest.emplace_back(key, value); + } + } +} + +// Clear all variables +inline void clear_vars(ordered_env_vars& vars) { + vars.clear(); +} + +} // namespace dropshell + +#endif diff --git a/source/src/utils/utils.cpp b/source/src/utils/utils.cpp index 45cd615..8b3c4a8 100644 --- a/source/src/utils/utils.cpp +++ b/source/src/utils/utils.cpp @@ -376,25 +376,24 @@ std::string replace_with_environment_variables_like_bash(std::string str) { } -std::string substitute_provided_key_value_pairs(std::string str, const std::map &env_vars) +std::string substitute_provided_key_value_pairs(std::string str, const ordered_env_vars& env_vars) { // Combined regex pattern for both ${var} and $var formats std::regex var_pattern("\\$(?:\\{([^}]+)\\}|([a-zA-Z0-9_]+))"); std::string result = str; std::smatch match; - + while (std::regex_search(result, match, var_pattern)) { // match[1] will contain capture from ${var} format // match[2] will contain capture from $var format std::string var_name = match[1].matched ? match[1].str() : match[2].str(); - - // Get value from environment variables map - auto it = env_vars.find(var_name); - std::string value = (it != env_vars.end()) ? it->second : ""; - + + // Get value from environment variables + std::string value = get_var(env_vars, var_name); + result = result.replace(match.position(), match.length(), value); } - + return result; } diff --git a/source/src/utils/utils.hpp b/source/src/utils/utils.hpp index a2c56e6..9165799 100644 --- a/source/src/utils/utils.hpp +++ b/source/src/utils/utils.hpp @@ -5,6 +5,7 @@ #include #include "output.hpp" +#include "ordered_env.hpp" #include namespace dropshell { @@ -54,7 +55,7 @@ std::string right_align(const std::string & str, int width); std::string center_align(const std::string & str, int width); std::string replace_with_environment_variables_like_bash(std::string str); -std::string substitute_provided_key_value_pairs(std::string str, const std::map & env_vars); +std::string substitute_provided_key_value_pairs(std::string str, const ordered_env_vars& env_vars); int get_console_width();