Files
dropshell/source/src/templates.cpp
j 8bf5583818
All checks were successful
Build-Test-Publish / build (linux/amd64) (push) Successful in 1m17s
Build-Test-Publish / build (linux/arm64) (push) Successful in 2m47s
fast check for autocomplete!
2025-09-30 13:56:47 +13:00

724 lines
30 KiB
C++

#include <filesystem>
#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <map>
#include <chrono>
#include <libassert/assert.hpp>
#include "utils/envmanager.hpp"
#include "utils/directories.hpp"
#include "utils/utils.hpp"
#include "utils/output.hpp"
#include "templates.hpp"
#include "config.hpp"
#include "utils/hash.hpp"
namespace dropshell {
// ------------------------------------------------------------------------------------------------
// template_source_local
// ------------------------------------------------------------------------------------------------
std::set<std::string> template_source_local::get_template_list() {
std::set<std::string> templates;
// Helper function to add templates from a directory
auto add_templates_from_dir = [&templates](const std::string& dir_path) {
if (!std::filesystem::exists(dir_path))
return;
for (const auto& entry : std::filesystem::directory_iterator(dir_path))
if (entry.is_directory())
templates.insert(entry.path().filename().string());
};
add_templates_from_dir(mLocalPath);
return templates;
}
bool template_source_local::has_template(const std::string& template_name) {
std::filesystem::path path = mLocalPath / template_name;
return (std::filesystem::exists(path));
}
bool template_source_local::template_command_exists(const std::string& template_name, const std::string& command) {
std::filesystem::path path = mLocalPath / template_name / (command+".sh");
return std::filesystem::exists(path);
}
template_info template_source_local::get_template_info(const std::string& template_name, bool fast) {
std::filesystem::path path = mLocalPath / template_name;
if (!std::filesystem::exists(path))
return template_info();
return template_info(
template_name,
mLocalPath.string(),
path
);
}
// ------------------------------------------------------------------------------------------------
// template_source_registry
// ------------------------------------------------------------------------------------------------
std::set<std::string> template_source_registry::get_template_list()
{
// Query the registry for available templates
// The registry should return a JSON list of available templates
std::string list_url = mRegistry.url + "/dir";
nlohmann::json json_response;
// For HTTPS URLs, use curl to fetch the JSON
if (list_url.substr(0, 8) == "https://") {
std::string temp_file = "/tmp/dropshell_registry_list_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count());
std::string cmd = "curl -fsSL " + quote(list_url) + " -o " + quote(temp_file) + " 2>/dev/null";
int result = system(cmd.c_str());
if (result == 0) {
try {
std::ifstream file(temp_file);
if (file.is_open()) {
file >> json_response;
file.close();
}
} catch (...) {
// Failed to parse JSON
}
std::filesystem::remove(temp_file);
}
} else {
json_response = get_json_from_url(list_url);
}
std::set<std::string> templates;
if (json_response.is_null() || !json_response.contains("entries")) {
warning << "Failed to get template list from registry: " << mRegistry.name << std::endl;
return templates;
}
// Parse the entries array to extract unique template names
// Only process entries that have labeltags (skip untagged files)
int total_entries = 0;
int skipped_entries = 0;
for (const auto& entry : json_response["entries"]) {
total_entries++;
if (entry.contains("labeltags") && entry["labeltags"].is_array() && !entry["labeltags"].empty()) {
for (const auto& label : entry["labeltags"]) {
// Extract template name from label (format: "template:version")
std::string label_str = label.get<std::string>();
size_t colon_pos = label_str.find(':');
if (colon_pos != std::string::npos) {
std::string template_name = label_str.substr(0, colon_pos);
templates.insert(template_name);
}
}
} else {
skipped_entries++;
// Entry has no labeltags or empty labeltags - skip it
debug << "Skipping registry entry without labeltags" << std::endl;
}
}
if (skipped_entries > 0) {
debug << "Registry " << mRegistry.name << ": Processed " << (total_entries - skipped_entries)
<< " tagged entries, skipped " << skipped_entries << " untagged entries" << std::endl;
}
return templates;
}
bool template_source_registry::has_template(const std::string& template_name)
{
// First check if we have it cached
std::filesystem::path cache_dir = get_cache_dir();
std::filesystem::path template_cache_dir = cache_dir / template_name;
if (std::filesystem::exists(template_cache_dir)) {
// Template found in cache
return true;
}
// Check if template exists in registry
std::string check_url = mRegistry.url + "/exists/" + template_name + ":latest";
// For HTTPS URLs, use curl to fetch the JSON
nlohmann::json json_response;
if (check_url.substr(0, 8) == "https://") {
// Create a temporary file for the response
std::string temp_file = "/tmp/dropshell_registry_check_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count());
std::string cmd = "curl -fsSL " + quote(check_url) + " -o " + quote(temp_file) + " 2>/dev/null";
int result = system(cmd.c_str());
if (result == 0) {
try {
std::ifstream file(temp_file);
if (file.is_open()) {
file >> json_response;
file.close();
}
} catch (const std::exception& e) {
warning << "Failed to parse JSON response from " << check_url << ": " << e.what() << std::endl;
}
std::filesystem::remove(temp_file);
} else {
// curl failed - network issue or server down
return std::filesystem::exists(template_cache_dir);
}
} else {
json_response = get_json_from_url(check_url);
}
if (!json_response.is_null() && json_response.contains("exists")) {
return json_response["exists"].get<bool>();
}
// If registry check failed but we have cache, use cache
return std::filesystem::exists(template_cache_dir);
}
template_info template_source_registry::get_template_info(const std::string& template_name, bool fast)
{
// Get cache directory
std::filesystem::path cache_dir = get_cache_dir();
std::filesystem::path template_cache_dir = cache_dir / template_name;
std::filesystem::path template_json_file = cache_dir / (template_name + ".json");
// Create cache directory if it doesn't exist
if (!std::filesystem::exists(cache_dir)) {
std::filesystem::create_directories(cache_dir);
}
// If we have a cached version and can't reach the registry, use the cache
bool have_cache = std::filesystem::exists(template_cache_dir) && std::filesystem::exists(template_json_file);
// Check if template exists (in cache or registry)
if (!has_template(template_name)) {
return template_info();
}
// fast: don't bother updating anything - if we have a cached version use that.
if (fast && have_cache) {
return template_info(
template_name,
"Registry: " + mRegistry.name + " (cached)",
template_cache_dir
);
}
// Get metadata from registry to check version
std::string meta_url = mRegistry.url + "/meta/" + template_name + ":latest";
nlohmann::json registry_metadata;
// For HTTPS URLs, use curl to fetch the JSON
if (meta_url.substr(0, 8) == "https://") {
std::string temp_file = "/tmp/dropshell_registry_meta_" + std::to_string(std::chrono::system_clock::now().time_since_epoch().count());
std::string cmd = "curl -fsSL " + quote(meta_url) + " -o " + quote(temp_file) + " 2>/dev/null";
int result = system(cmd.c_str());
if (result == 0) {
try {
std::ifstream file(temp_file);
if (file.is_open()) {
file >> registry_metadata;
file.close();
}
} catch (...) {
// Failed to parse JSON
}
std::filesystem::remove(temp_file);
}
} else {
registry_metadata = get_json_from_url(meta_url);
}
if (registry_metadata.is_null()) {
// If we can't get metadata from registry but have cache, use cache
if (have_cache) {
info << "Registry unavailable, using cached template: " << template_name << std::endl;
return template_info(
template_name,
"Registry: " + mRegistry.name + " (cached)",
template_cache_dir
);
}
warning << "Failed to get metadata for template: " << template_name << std::endl;
return template_info();
}
// Check if we need to download/update the template
bool need_download = true;
std::string registry_version = "unknown";
std::string registry_hash = "";
// Extract version and hash from registry metadata
if (registry_metadata.contains("metadata")) {
auto& metadata = registry_metadata["metadata"];
if (metadata.contains("version")) {
registry_version = metadata["version"].get<std::string>();
}
// REQUIRED: unpackedhash - the hash of extracted contents
if (metadata.contains("unpackedhash")) {
registry_hash = metadata["unpackedhash"].get<std::string>();
//debug << "Found unpackedhash in metadata: " << registry_hash << std::endl;
} else {
// unpackedhash is required for security
error << "Template '" << template_name << "' from registry '" << mRegistry.name
<< "' does not provide unpackedhash for integrity verification." << std::endl;
error << "This template cannot be downloaded for security reasons." << std::endl;
error << "Please contact the registry administrator to update the template metadata." << std::endl;
return template_info();
}
}
// Check if we have a cached version
if (std::filesystem::exists(template_json_file)) {
try {
std::ifstream cache_file(template_json_file);
nlohmann::json cache_json;
cache_file >> cache_json;
cache_file.close();
// Compare versions or hashes
if (cache_json.contains("hash") && !registry_hash.empty()) {
if (cache_json["hash"].get<std::string>() == registry_hash) {
need_download = false;
}
} else if (cache_json.contains("version")) {
std::string cached_version = cache_json["version"].get<std::string>();
if (cached_version == registry_version) {
need_download = false;
}
}
} catch (...) {
// If reading cache fails, re-download
need_download = true;
}
}
// Download and extract if needed
if (need_download) {
info << "Downloading template '" << template_name << "' from registry..." << std::endl;
// Download the .tgz file
std::string download_url = mRegistry.url + "/" + template_name + ":latest";
std::filesystem::path temp_tgz = cache_dir / (template_name + ".tgz");
if (!download_file(download_url, temp_tgz.string())) {
error << "Failed to download template: " << template_name << std::endl;
return template_info();
}
// Remove old template directory if it exists
if (std::filesystem::exists(template_cache_dir)) {
std::filesystem::remove_all(template_cache_dir);
}
// Extract the .tgz file
std::string extract_cmd = "tar -xzf " + quote(temp_tgz.string()) + " -C " + quote(cache_dir.string());
int result = system(extract_cmd.c_str());
if (result != 0) {
error << "Failed to extract template: " << template_name << std::endl;
std::filesystem::remove(temp_tgz);
return template_info();
}
// Clean up the .tgz file
std::filesystem::remove(temp_tgz);
// Calculate actual hash of extracted template
std::string actual_hash = hash_directory_recursive(template_cache_dir.string());
// Verify the extracted template hash matches what registry claimed
// unpackedhash is required, so registry_hash should always be set here
if (registry_hash.empty()) {
// This shouldn't happen as we check for it above, but handle it just in case
error << "Internal error: unpackedhash was not properly set" << std::endl;
std::filesystem::remove_all(template_cache_dir);
return template_info();
}
if (actual_hash != registry_hash) {
error << "Template hash verification failed!" << std::endl;
error << "Expected hash: " << registry_hash << std::endl;
error << "Actual hash: " << actual_hash << std::endl;
error << "The downloaded template '" << template_name << "' may be corrupted or tampered with." << std::endl;
// Remove the corrupted template
std::filesystem::remove_all(template_cache_dir);
return template_info();
}
info << "Template extracted successfully. SHA256: " << actual_hash << std::endl;
info << "Note: Hash verification temporarily disabled during migration from XXHash64 to SHA256" << std::endl;
// Generate .template_info.env if it doesn't exist
std::filesystem::path template_info_env_path = template_cache_dir / "config" / filenames::template_info_env;
if (!std::filesystem::exists(template_info_env_path)) {
// Create config directory if needed
std::filesystem::create_directories(template_cache_dir / "config");
// Write .template_info.env file
std::ofstream info_file(template_info_env_path);
info_file << "# Template information" << std::endl;
info_file << "TEMPLATE=" << template_name << std::endl;
info_file << "TEMPLATE_SOURCE=registry" << std::endl;
info_file << "TEMPLATE_REGISTRY=" << mRegistry.name << std::endl;
info_file << "TEMPLATE_VERSION=" << registry_version << std::endl;
// Always write the actual calculated hash
info_file << "TEMPLATE_HASH=" << actual_hash << std::endl;
info_file.close();
}
// Update cache JSON file
nlohmann::json cache_json;
cache_json["template"] = template_name;
cache_json["version"] = registry_version;
cache_json["hash"] = actual_hash; // Store actual calculated hash
cache_json["registry"] = mRegistry.name;
cache_json["last_updated"] = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
std::ofstream cache_file(template_json_file);
cache_file << cache_json.dump(4);
cache_file.close();
info << "Template '" << template_name << "' downloaded and cached successfully" << std::endl;
}
// Return template info pointing to the cached template
return template_info(
template_name,
"Registry: " + mRegistry.name,
template_cache_dir
);
}
bool template_source_registry::template_command_exists(const std::string& template_name, const std::string& command)
{
// Get template info to ensure it's downloaded and cached
auto tinfo = get_template_info(template_name, false);
if (!tinfo.is_set()) {
return false;
}
// Check if the command script exists in the cached template
std::filesystem::path script_path = tinfo.local_template_path() / (command + ".sh");
return std::filesystem::exists(script_path);
}
std::filesystem::path template_source_registry::get_cache_dir()
{
return localpath::template_cache();
}
// ------------------------------------------------------------------------------------------------
// template_manager
// ------------------------------------------------------------------------------------------------
void template_manager::list_templates() const {
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
auto templates = get_template_list();
if (templates.empty()) {
std::cout << "No templates found." << std::endl;
return;
}
std::cout << "Available templates:" << std::endl;
// print templates.
std::cout << std::string(60, '-') << std::endl;
bool first = true;
for (const auto& t : templates) {
std::cout << (first ? "" : ", ") << t;
first = false;
}
std::cout << std::endl;
std::cout << std::string(60, '-') << std::endl;
}
std::set<std::string> template_manager::get_template_list() const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
std::set<std::string> templates;
for (const auto& source : mSources) {
auto source_templates = source->get_template_list();
templates.insert(source_templates.begin(), source_templates.end());
}
return templates;
}
bool template_manager::has_template(const std::string &template_name) const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
template_source_interface* source = get_source(template_name);
if (!source)
return false;
return true;
}
template_info template_manager::get_template_info(const std::string &template_name, bool fast) const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
template_source_interface* source = get_source(template_name);
if (source)
return source->get_template_info(template_name, fast);
// fail
return template_info();
}
bool template_manager::template_command_exists(const std::string &template_name, const std::string &command) const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
template_source_interface* source = get_source(template_name);
if (!source) {
error << "Template '" << template_name << "' not found" << std::endl;
return false;
}
return source->template_command_exists(template_name, command);
}
bool template_manager::create_template(const std::string &template_name) const
{
if (!legal_service_name(template_name)) {
error << "Template name contains illegal characters: " << template_name << std::endl;
return false;
}
// 1. Create a new directory in the user templates directory
std::vector<std::string> local_server_definition_paths = gConfig().get_local_server_definition_paths();
if (local_server_definition_paths.empty()) {
error << "No local server definition paths found" << std::endl;
info << "Run 'dropshell edit' to configure DropShell" << std::endl;
return false;
}
auto tinfo = get_template_info(template_name, false);
if (tinfo.is_set()) {
error << "Template '" << template_name << "' already exists at " << tinfo.locationID() << std::endl;
return false;
}
auto local_template_paths = gConfig().get_local_template_paths();
if (local_template_paths.empty()) {
error << "No local template paths found" << std::endl;
info << "Run 'dropshell edit' to add one to the DropShell config" << std::endl;
return false;
}
std::string new_template_path = local_template_paths[0] + "/" + template_name;
// Create the new template directory
std::filesystem::create_directories(new_template_path);
// 2. Copy the example template from the system templates directory
auto example_info = gTemplateManager().get_template_info("example-nginx");
if (!example_info.is_set()) {
error << "Example template not found" << std::endl;
return false;
}
std::string example_template_path = example_info.local_template_path();
// Copy all files from example template to new template
for (const auto& entry : std::filesystem::recursive_directory_iterator(example_template_path)) {
std::string relative_path = entry.path().string().substr(example_template_path.length());
std::string target_path = new_template_path + relative_path;
if (entry.is_directory()) {
std::filesystem::create_directory(target_path);
} else {
std::filesystem::copy_file(entry.path(), target_path);
}
}
// modify the TEMPLATE=example line in the .template_info.env file to TEMPLATE=<template_name>
std::string search_string = "TEMPLATE=";
std::string replacement_line = "TEMPLATE=" + template_name;
std::string service_env_path = new_template_path + "/config/" + filenames::template_info_env;
if (!replace_line_in_file(service_env_path, search_string, replacement_line)) {
error << "Failed to replace TEMPLATE= line in the " << filenames::template_info_env <<" file" << std::endl;
return false;
}
// 3. Print out the README.txt file and the path
std::string readme_path = new_template_path + "/README.txt";
if (std::filesystem::exists(readme_path)) {
std::cout << "\nREADME contents:" << std::endl;
std::cout << std::string(60, '-') << std::endl;
std::ifstream readme_file(readme_path);
if (readme_file.is_open()) {
std::string line;
while (std::getline(readme_file, line)) {
std::cout << line << std::endl;
}
readme_file.close();
}
std::cout << std::string(60, '-') << std::endl;
}
std::cout << std::endl;
std::cout << "Template '" << template_name << "' created at " << new_template_path << std::endl;
return test_template(new_template_path);
}
void template_manager::load_sources()
{
ASSERT(mSources.empty(), "Template manager already loaded (sources are not empty).");
ASSERT(gConfig().is_config_set(), "Config not set.");
ASSERT(!mLoaded, "Template manager already loaded.");
// Add local template sources only if the paths exist and are directories
auto local_template_paths = gConfig().get_local_template_paths();
for (const auto& path : local_template_paths) {
if (std::filesystem::exists(path) && std::filesystem::is_directory(path)) {
mSources.push_back(std::make_unique<template_source_local>(path));
} else {
info << "Skipping non-existent or invalid local template path: " << path << std::endl;
}
}
// Add registry sources - these should always be added
std::vector<tRegistryEntry> registry_urls = gConfig().get_template_registry_urls();
for (const tRegistryEntry & url : registry_urls) {
mSources.push_back(std::make_unique<template_source_registry>(url));
}
mLoaded = true;
}
void template_manager::print_sources() const
{
std::cout << "Template sources: ";
for (const auto& source : mSources) {
std::cout << "[" << source->get_description() << "] ";
}
std::cout << std::endl;
}
bool template_manager::required_file(std::string path, std::string template_name)
{
if (!std::filesystem::exists(path)) {
error << path << " file not found in template - REQUIRED." << template_name << std::endl;
return false;
}
return true;
}
template_source_interface *template_manager::get_source(const std::string &template_name) const
{
ASSERT(mLoaded && mSources.size() > 0, "Template manager not loaded, or no template sources found.");
for (const auto& source : mSources) {
if (source->has_template(template_name)) {
return source.get();
}
}
return nullptr;
}
bool template_manager::test_template(const std::string &template_path)
{
if (template_path.empty())
return false;
if (!std::filesystem::exists(template_path))
return false;
std::string template_name = std::filesystem::path(template_path).filename().string();
std::vector<std::string> required_files = {
"config/" + filenames::service_env,
"config/" + filenames::template_info_env,
"install.sh",
"uninstall.sh"
};
for (const auto& file : required_files) {
if (!required_file(template_path + "/" + file, template_name))
return false;
// check if file is executable, if it ends in .sh
std::string suffix=".sh";
if (file.find(suffix) == file.size() - suffix.size())
{
std::filesystem::path path = template_path + "/" + file;
auto perms = std::filesystem::status(path).permissions();
if ((perms & std::filesystem::perms::owner_exec) == std::filesystem::perms::none)
error << file << " is not executable" << std::endl;
}
}
// ------------------------------------------------------------
// check TEMPLATE= line.
std::map<std::string, std::string> all_env_vars;
std::vector<std::string> env_files = {
"config/" + filenames::service_env,
"config/" + filenames::template_info_env
};
for (const auto& file : env_files) {
{ // load service.env from the service on this machine.
std::map<std::string, std::string> env_vars;
envmanager env_manager(template_path + "/" + file);
env_manager.load();
env_manager.get_all_variables(env_vars);
all_env_vars.merge(env_vars);
}
}
// determine template name.
auto it = all_env_vars.find("TEMPLATE");
if (it == all_env_vars.end()) {
error << "TEMPLATE variable not found in " << template_path << std::endl;
return false;
}
std::string env_template_name = it->second;
if (env_template_name.empty()) {
error << "TEMPLATE variable is empty in " << template_path << std::endl;
return false;
}
if (env_template_name != template_name) {
error << "TEMPLATE variable is wrong in " << template_path << std::endl;
return false;
}
return true;
}
template_manager & gTemplateManager()
{
static template_manager instance;
return instance;
}
template_info::template_info(const std::string &template_name, const std::string &location_id, const std::filesystem::path &local_template_path) :
mTemplateName(template_name),
mLocationID(location_id),
mTemplateLocalPath(local_template_path),
mTemplateValid(template_manager::test_template(local_template_path.string())),
mIsSet(!template_name.empty() && !location_id.empty() && !local_template_path.empty()),\
mHash("")
{
if (!std::filesystem::exists(local_template_path))
{
error << "Template path does not exist: " << local_template_path << std::endl;
return;
}
mHash = hash_directory_recursive(local_template_path);
}
} // namespace dropshell