Files
dropshell/source/src/templates.cpp
j 63fdc2c0cb
All checks were successful
Build-Test-Publish / build (linux/amd64) (push) Successful in 40s
Build-Test-Publish / build (linux/arm64) (push) Successful in 2m45s
feat: Add add-template command to register local template sources
2026-03-23 09:56:23 +13:00

704 lines
28 KiB
C++

#include <filesystem>
#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <map>
#include <chrono>
#include <libassert/assert.hpp>
#include "utils/envmanager.hpp"
#include "utils/directories.hpp"
#include "utils/utils.hpp"
#include "utils/output.hpp"
#include "utils/execute.hpp"
#include "templates.hpp"
#include "config.hpp"
#include "utils/hash.hpp"
namespace dropshell {
// ------------------------------------------------------------------------------------------------
// template_manager — loading and resolution
// ------------------------------------------------------------------------------------------------
void template_manager::load_sources()
{
ASSERT(mSources.empty(), "Template manager already loaded (sources are not empty).");
ASSERT(gConfig().is_config_set(), "Config not set.");
ASSERT(!mLoaded, "Template manager already loaded.");
// Read template_paths.json from each server_definition_path
auto server_def_paths = gConfig().get_local_server_definition_paths();
for (const auto& sdp : server_def_paths) {
std::filesystem::path json_path = std::filesystem::path(sdp) / filenames::template_paths_json;
if (!std::filesystem::exists(json_path))
continue;
std::ifstream f(json_path);
if (!f.is_open()) {
warning << "Could not open " << json_path << std::endl;
continue;
}
nlohmann::json j;
try {
j = nlohmann::json::parse(f);
} catch (nlohmann::json::parse_error& ex) {
error << "Failed to parse " << json_path << ": " << ex.what() << std::endl;
continue;
}
if (!j.is_array()) {
error << json_path << " must be a JSON array" << std::endl;
continue;
}
for (const auto& entry : j) {
if (!entry.is_string() || entry.get<std::string>().empty())
continue;
std::string s = entry.get<std::string>();
TemplateSource src;
// Split on first ':' — Linux paths never contain colons
size_t colon = s.find(':');
if (colon != std::string::npos) {
src.local_path = s.substr(0, colon);
src.git_url = s.substr(colon + 1);
} else {
src.local_path = s;
}
mSources.push_back(src);
}
}
// Resolve templates from all sources
resolve_templates();
mLoaded = true;
}
void template_manager::resolve_templates()
{
mTemplateMap.clear();
for (size_t i = 0; i < mSources.size(); ++i)
resolve_source(i);
}
void template_manager::resolve_source(size_t source_index)
{
const auto& src = mSources[source_index];
if (!std::filesystem::exists(src.local_path) || !std::filesystem::is_directory(src.local_path)) {
if (!src.git_url.empty())
debug << "Source path does not exist (run 'dropshell pull' to clone): " << src.local_path << std::endl;
else
warning << "Source path does not exist: " << src.local_path << std::endl;
return;
}
// Try to read dropshell-templates.list
std::filesystem::path list_file = std::filesystem::path(src.local_path) / filenames::dropshell_templates_list;
std::vector<std::string> template_dirs;
if (std::filesystem::exists(list_file)) {
template_dirs = parse_list_file(src.local_path);
} else {
// Discover templates by searching for template_info.env
template_dirs = discover_templates(src.local_path);
// Write the discovered list so next time is fast
if (!template_dirs.empty())
write_list_file(src.local_path, template_dirs);
}
// Remove stale entries for this source from mTemplateMap before re-adding
for (auto it = mTemplateMap.begin(); it != mTemplateMap.end(); ) {
if (it->second.source_index == source_index)
it = mTemplateMap.erase(it);
else
++it;
}
// Add to map (first-wins across sources)
for (const auto& tdir : template_dirs) {
std::filesystem::path full_path = std::filesystem::path(src.local_path) / tdir;
if (!std::filesystem::exists(full_path) || !std::filesystem::is_directory(full_path)) {
debug << "Template directory does not exist: " << full_path << std::endl;
continue;
}
std::string name = full_path.filename().string();
if (mTemplateMap.find(name) == mTemplateMap.end()) {
mTemplateMap[name] = ResolvedTemplate{full_path, source_index};
}
}
}
std::vector<std::string> template_manager::parse_list_file(const std::string& local_path) const
{
std::vector<std::string> result;
std::filesystem::path list_file = std::filesystem::path(local_path) / filenames::dropshell_templates_list;
std::ifstream f(list_file);
if (!f.is_open())
return result;
std::string line;
while (std::getline(f, line)) {
// Trim whitespace
size_t start = line.find_first_not_of(" \t\r\n");
if (start == std::string::npos) continue;
size_t end = line.find_last_not_of(" \t\r\n");
line = line.substr(start, end - start + 1);
// Skip comments and blank lines
if (line.empty() || line[0] == '#') continue;
result.push_back(line);
}
return result;
}
std::vector<std::string> template_manager::discover_templates(const std::string& local_path) const
{
std::vector<std::string> result;
std::filesystem::path root(local_path);
for (auto it = std::filesystem::recursive_directory_iterator(root, std::filesystem::directory_options::skip_permission_denied);
it != std::filesystem::recursive_directory_iterator(); ++it)
{
if (!it->is_directory()) continue;
// Skip .git directories
if (it->path().filename() == ".git") {
it.disable_recursion_pending();
continue;
}
// Check if this directory contains template_info.env
std::error_code ec;
if (std::filesystem::exists(it->path() / filenames::template_info_env, ec) && !ec) {
// Get relative path from root
std::string rel = std::filesystem::relative(it->path(), root).string();
result.push_back(rel);
// Don't recurse into template directories (templates don't nest)
it.disable_recursion_pending();
}
}
return result;
}
void template_manager::write_list_file(const std::string& local_path, const std::vector<std::string>& template_dirs) const
{
std::filesystem::path list_file = std::filesystem::path(local_path) / filenames::dropshell_templates_list;
std::ofstream f(list_file);
if (!f.is_open()) {
debug << "Could not write " << list_file << std::endl;
return;
}
f << "# Auto-generated by dropshell — template directories relative to this file" << std::endl;
for (const auto& dir : template_dirs)
f << dir << std::endl;
info << "Wrote " << list_file << " (" << template_dirs.size() << " templates)" << std::endl;
}
// ------------------------------------------------------------------------------------------------
// template_manager — git operations
// ------------------------------------------------------------------------------------------------
bool template_manager::git_clone_source(const TemplateSource& source)
{
if (source.git_url.empty()) return false;
info << "Cloning " << source.git_url << "" << source.local_path << std::endl;
// Create parent directory if needed
std::filesystem::path parent = std::filesystem::path(source.local_path).parent_path();
if (!std::filesystem::exists(parent))
std::filesystem::create_directories(parent);
std::string cmd = "git clone " + quote(source.git_url) + " " + quote(source.local_path) + " 2>&1";
std::string output;
ordered_env_vars empty_env;
bool ok = execute_local_command(".", cmd, empty_env, &output, cMode::Silent);
if (!ok)
error << "git clone failed: " << output << std::endl;
else
info << "Cloned successfully." << std::endl;
return ok;
}
bool template_manager::git_pull_source(const TemplateSource& source)
{
if (!std::filesystem::exists(std::filesystem::path(source.local_path) / ".git"))
return false;
// Verify remote URL matches if git_url is set
if (!source.git_url.empty()) {
std::string remote_output;
ordered_env_vars empty_env;
std::string check_cmd = "git -C " + quote(source.local_path) + " remote get-url origin 2>/dev/null";
execute_local_command(".", check_cmd, empty_env, &remote_output, cMode::Silent);
// Trim
while (!remote_output.empty() && (remote_output.back() == '\n' || remote_output.back() == '\r'))
remote_output.pop_back();
if (!remote_output.empty() && remote_output != source.git_url) {
warning << "Remote URL mismatch for " << source.local_path << std::endl;
warning << " Expected: " << source.git_url << std::endl;
warning << " Actual: " << remote_output << std::endl;
return false;
}
}
debug << "Pulling " << source.local_path << std::endl;
std::string cmd = "git -C " + quote(source.local_path) + " pull --ff-only 2>&1";
std::string output;
ordered_env_vars empty_env;
bool ok = execute_local_command(".", cmd, empty_env, &output, cMode::Silent);
if (!ok)
warning << "git pull failed for " << source.local_path << ": " << output << std::endl;
return ok;
}
bool template_manager::pull_all()
{
bool any_changes = false;
for (const auto& src : mSources) {
if (!std::filesystem::exists(src.local_path)) {
if (!src.git_url.empty()) {
if (git_clone_source(src))
any_changes = true;
} else {
warning << "Source path does not exist and no git URL configured: " << src.local_path << std::endl;
}
} else if (std::filesystem::exists(std::filesystem::path(src.local_path) / ".git")) {
if (git_pull_source(src))
any_changes = true;
}
// else: local-only, no .git — skip
}
// Re-resolve all templates after pulling
resolve_templates();
return true;
}
bool template_manager::pull_for_template(const std::string& template_name)
{
auto it = mTemplateMap.find(template_name);
if (it == mTemplateMap.end())
return false;
const auto& src = mSources[it->second.source_index];
if (src.git_url.empty())
return true; // local-only, nothing to pull
if (!std::filesystem::exists(std::filesystem::path(src.local_path) / ".git"))
return true; // no .git dir, nothing to pull
git_pull_source(src);
// Re-resolve this source's templates
resolve_source(it->second.source_index);
return true;
}
// ------------------------------------------------------------------------------------------------
// template_manager — queries
// ------------------------------------------------------------------------------------------------
void template_manager::list_templates() const {
ASSERT(mLoaded, "Template manager not loaded.");
auto templates = get_template_list();
if (templates.empty()) {
std::cout << "No templates found." << std::endl;
return;
}
std::cout << "Available templates:" << std::endl;
std::cout << std::string(60, '-') << std::endl;
bool first = true;
for (const auto& t : templates) {
std::cout << (first ? "" : ", ") << t;
first = false;
}
std::cout << std::endl;
std::cout << std::string(60, '-') << std::endl;
}
std::set<std::string> template_manager::get_template_list() const
{
ASSERT(mLoaded, "Template manager not loaded.");
std::set<std::string> templates;
for (const auto& [name, _] : mTemplateMap)
templates.insert(name);
return templates;
}
std::vector<std::pair<std::string, std::string>> template_manager::get_template_list_with_source() const
{
ASSERT(mLoaded, "Template manager not loaded.");
std::vector<std::pair<std::string, std::string>> result;
for (const auto& [name, resolved] : mTemplateMap) {
// Only include templates that have install.sh
if (!std::filesystem::exists(resolved.template_dir / "install.sh"))
continue;
const auto& src = mSources[resolved.source_index];
std::string desc = src.git_url.empty()
? "Local: " + src.local_path
: "Git: " + src.local_path;
result.push_back({name, desc});
}
return result;
}
bool template_manager::has_template(const std::string &template_name) const
{
ASSERT(mLoaded, "Template manager not loaded.");
return mTemplateMap.count(template_name) > 0;
}
template_info template_manager::get_template_info(const std::string &template_name) const
{
ASSERT(mLoaded, "Template manager not loaded.");
auto it = mTemplateMap.find(template_name);
if (it == mTemplateMap.end())
return template_info();
const auto& src = mSources[it->second.source_index];
return template_info(template_name, src.local_path, it->second.template_dir);
}
bool template_manager::template_command_exists(const std::string &template_name, const std::string &command) const
{
ASSERT(mLoaded, "Template manager not loaded.");
auto it = mTemplateMap.find(template_name);
if (it == mTemplateMap.end()) {
error << "Template '" << template_name << "' not found" << std::endl;
return false;
}
return std::filesystem::exists(it->second.template_dir / (command + ".sh"));
}
// ------------------------------------------------------------------------------------------------
// template_manager — create template
// ------------------------------------------------------------------------------------------------
// Helper function to write a file with content
static bool write_template_file(const std::string& path, const std::string& content, bool executable = false) {
std::ofstream file(path);
if (!file.is_open()) {
error << "Failed to create file: " << path << std::endl;
return false;
}
file << content;
file.close();
if (executable) {
std::filesystem::permissions(path,
std::filesystem::perms::owner_all |
std::filesystem::perms::group_read | std::filesystem::perms::group_exec |
std::filesystem::perms::others_read | std::filesystem::perms::others_exec);
}
return true;
}
bool template_manager::create_template(const std::string &template_name) const
{
if (!legal_service_name(template_name)) {
error << "Template name contains illegal characters: " << template_name << std::endl;
return false;
}
// 1. Check template doesn't already exist
auto tinfo = get_template_info(template_name);
if (tinfo.is_set()) {
error << "Template '" << template_name << "' already exists at " << tinfo.locationID() << std::endl;
return false;
}
// 2. Determine where to create the template
if (mSources.empty()) {
error << "No template sources configured" << std::endl;
info << "Create a template_paths.json in your server definition path" << std::endl;
return false;
}
std::string base_path = mSources[0].local_path;
if (!std::filesystem::exists(base_path)) {
error << "Template source path does not exist: " << base_path << std::endl;
return false;
}
std::string new_template_path = base_path + "/" + template_name;
// 3. Create directory structure
std::filesystem::create_directories(new_template_path + "/config");
// 4. Generate template files
std::string template_info_env = R"TMPL(# Template metadata - DO NOT EDIT
# This file is replaced when the template is updated.
# Requirements
REQUIRES_HOST_ROOT=false
REQUIRES_DOCKER=true
REQUIRES_DOCKER_ROOT=false
# Docker image settings
IMAGE_REGISTRY="docker.io"
IMAGE_REPO="nginx"
IMAGE_TAG="alpine"
# Volume definitions
DATA_VOLUME="${CONTAINER_NAME}_data"
)TMPL";
if (!write_template_file(new_template_path + "/" + filenames::template_info_env, template_info_env)) return false;
std::string service_env = R"TMPL(# Service identification (REQUIRED)
CONTAINER_NAME=)TMPL" + template_name + R"TMPL(
# Server settings (REQUIRED by dropshell)
SSH_USER="root"
# Service-specific settings
HTTP_PORT=8080
)TMPL";
if (!write_template_file(new_template_path + "/config/" + filenames::service_env, service_env)) return false;
std::string install_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
_check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG"
_check_docker_installed || _die "Docker test failed"
docker pull -q "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || _die "Failed to pull image"
bash "$SCRIPT_DIR/stop.sh" 2>/dev/null || true
_remove_container "$CONTAINER_NAME" 2>/dev/null || true
bash "$SCRIPT_DIR/start.sh" || _die "Failed to start container"
echo "Installation of ${CONTAINER_NAME} complete"
)BASH";
if (!write_template_file(new_template_path + "/install.sh", install_sh, true)) return false;
std::string uninstall_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
_check_required_env_vars "CONTAINER_NAME"
bash "$SCRIPT_DIR/stop.sh" 2>/dev/null || true
_remove_container "$CONTAINER_NAME" || _die "Failed to remove container"
echo "Uninstallation of ${CONTAINER_NAME} complete"
echo "Note: Data volumes have been preserved. To remove all data, use destroy.sh"
)BASH";
if (!write_template_file(new_template_path + "/uninstall.sh", uninstall_sh, true)) return false;
std::string start_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
_check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" "HTTP_PORT"
docker volume create "$DATA_VOLUME" 2>/dev/null || true
docker run -d \
--name "$CONTAINER_NAME" \
--restart unless-stopped \
-p "${HTTP_PORT}:80" \
-v "${DATA_VOLUME}:/usr/share/nginx/html" \
"$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || _die "Failed to start container"
echo "Container ${CONTAINER_NAME} started on port ${HTTP_PORT}"
)BASH";
if (!write_template_file(new_template_path + "/start.sh", start_sh, true)) return false;
std::string stop_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
_check_required_env_vars "CONTAINER_NAME"
docker stop "$CONTAINER_NAME" 2>/dev/null || true
echo "Container ${CONTAINER_NAME} stopped"
)BASH";
if (!write_template_file(new_template_path + "/stop.sh", stop_sh, true)) return false;
std::string status_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
_check_required_env_vars "CONTAINER_NAME"
if docker ps --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then
echo "Running"
else
echo "Stopped"
fi
)BASH";
if (!write_template_file(new_template_path + "/status.sh", status_sh, true)) return false;
std::string logs_sh = R"BASH(#!/bin/bash
source "${AGENT_PATH}/common.sh"
_check_required_env_vars "CONTAINER_NAME"
docker logs "$CONTAINER_NAME" "$@"
)BASH";
if (!write_template_file(new_template_path + "/logs.sh", logs_sh, true)) return false;
// 5. Update dropshell-templates.list
std::filesystem::path list_file = std::filesystem::path(base_path) / filenames::dropshell_templates_list;
std::ofstream lf(list_file, std::ios::app);
if (lf.is_open()) {
lf << template_name << std::endl;
lf.close();
}
// 6. Print summary
std::cout << "\nTemplate '" << template_name << "' created at " << new_template_path << std::endl;
std::cout << std::string(60, '-') << std::endl;
std::cout << "Next steps:" << std::endl;
std::cout << " 1. Edit " << new_template_path << "/config/service.env" << std::endl;
std::cout << " 2. Customize the scripts for your application" << std::endl;
std::cout << " 3. Run: dropshell validate " << new_template_path << std::endl;
std::cout << std::string(60, '-') << std::endl;
return test_template(new_template_path);
}
// ------------------------------------------------------------------------------------------------
// template_manager — sources display
// ------------------------------------------------------------------------------------------------
void template_manager::print_sources() const
{
std::cout << "Template sources:" << std::endl;
for (const auto& src : mSources) {
std::cout << " [" << src.local_path;
if (!src.git_url.empty())
std::cout << "" << src.git_url;
std::cout << "]" << std::endl;
}
}
// ------------------------------------------------------------------------------------------------
// template_manager — validation (static)
// ------------------------------------------------------------------------------------------------
bool template_manager::required_file(std::string path, std::string template_name)
{
if (!std::filesystem::exists(path)) {
error << path << " file not found in template - REQUIRED." << template_name << std::endl;
return false;
}
return true;
}
bool template_manager::test_template(const std::string &template_path)
{
if (template_path.empty())
return false;
if (!std::filesystem::exists(template_path))
return false;
std::string template_name = std::filesystem::path(template_path).filename().string();
std::vector<std::string> required_files = {
"config/" + filenames::service_env,
filenames::template_info_env,
"install.sh",
"uninstall.sh",
"status.sh"
};
for (const auto& file : required_files) {
if (!required_file(template_path + "/" + file, template_name))
return false;
std::string suffix=".sh";
if (file.find(suffix) == file.size() - suffix.size())
{
std::filesystem::path path = template_path + "/" + file;
auto perms = std::filesystem::status(path).permissions();
if ((perms & std::filesystem::perms::owner_exec) == std::filesystem::perms::none)
error << file << " is not executable" << std::endl;
}
}
// check required variables in template_info.env
ordered_env_vars all_env_vars;
std::vector<std::string> env_files = {
"config/" + filenames::service_env,
filenames::template_info_env
};
for (const auto& file : env_files) {
ordered_env_vars env_vars;
envmanager env_manager(template_path + "/" + file);
env_manager.load();
env_manager.get_all_variables(env_vars);
merge_vars(all_env_vars, env_vars);
}
std::vector<std::string> required_vars = {
"REQUIRES_HOST_ROOT",
"REQUIRES_DOCKER",
"REQUIRES_DOCKER_ROOT"
};
for (const auto & required_var : required_vars)
{
auto it = find_var(all_env_vars, required_var);
if (it == all_env_vars.end()) {
error << "Required variable "<< required_var<<" not defined in " << template_path << std::endl;
return false;
}
}
return true;
}
bool template_manager::check_template_shell_scripts_syntax(const std::string &template_path)
{
if (template_path.empty() || !std::filesystem::exists(template_path))
return false;
bool all_passed = true;
for (const auto& entry : std::filesystem::recursive_directory_iterator(template_path)) {
if (entry.is_regular_file() && entry.path().extension() == ".sh") {
std::string script_path = entry.path().string();
std::string command = "bash -n " + script_path + " 2>&1";
FILE* pipe = popen(command.c_str(), "r");
if (!pipe) {
error << "Failed to run bash -n on " << entry.path().filename() << std::endl;
all_passed = false;
continue;
}
char buffer[256];
std::string output;
while (fgets(buffer, sizeof(buffer), pipe) != nullptr)
output += buffer;
int result = pclose(pipe);
if (result != 0 || !output.empty()) {
error << "Syntax error in " << entry.path().filename() << ":" << std::endl;
error << output << std::endl;
all_passed = false;
}
}
}
return all_passed;
}
// ------------------------------------------------------------------------------------------------
// Singletons and template_info
// ------------------------------------------------------------------------------------------------
template_manager & gTemplateManager()
{
static template_manager instance;
return instance;
}
template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path) :
mTemplateName(template_name),
mLocationID(location_id),
mTemplateLocalPath(local_template_path),
mTemplateValid(template_manager::test_template(local_template_path.string())),
mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty())
{
if (!std::filesystem::exists(local_template_path))
{
error << "Template path does not exist: " << local_template_path << std::endl;
return;
}
}
std::filesystem::path template_info::local_template_service_env_path()
{
return mTemplateLocalPath / "config" / filenames::service_env;
}
std::filesystem::path template_info::local_template_info_env_path()
{
return mTemplateLocalPath / filenames::template_info_env;
}
} // namespace dropshell