.

.gitea/workflows/test.yaml

@@ -0,0 +1,23 @@
+name: Dropshell Test
+run-name: Test dropshell
+on: [push]
+
+jobs:
+  Build_and_Test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y openssh-server
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - name: Build
+        run: |
+          cd ${{ gitea.workspace }}/docker
+          ./compile.sh
+      - name: Test
+        run: |
+          cd ${{ gitea.workspace }}/docker/output
+          ./dropshell_x86_64 list
+          ./dropshell_x86_64 help

.gitignore

@@ -4,6 +4,7 @@ cmake-build-*/
 out/
 bin/
 lib/
+output/
 
 # Compiled Object files
 *.o

CMakeLists.txt

@@ -17,7 +17,9 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -DNDEBUG")
 string(TIMESTAMP CURRENT_YEAR "%Y")
 string(TIMESTAMP CURRENT_MONTH "%m")
 string(TIMESTAMP CURRENT_DAY "%d")
-set(PROJECT_VERSION "${CURRENT_YEAR}.${CURRENT_MONTH}.${CURRENT_DAY}")
+string(TIMESTAMP CURRENT_HOUR "%H")
+string(TIMESTAMP CURRENT_MINUTE "%M")
+set(PROJECT_VERSION "${CURRENT_YEAR}.${CURRENT_MONTH}${CURRENT_DAY}.${CURRENT_HOUR}${CURRENT_MINUTE}")
 string(TIMESTAMP RELEASE_DATE "%Y-%m-%d")
 
 # Configure version.hpp file
@@ -30,10 +32,6 @@ configure_file(
 # Set CMAKE_MODULE_PATH to include our custom find modules
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
-# Find required packages
-find_package(TBB REQUIRED)
-find_package(xxHash REQUIRED)
-
 # Auto-detect source files
 file(GLOB_RECURSE SOURCES "src/*.cpp")
 file(GLOB_RECURSE HEADERS "src/*.hpp")
@@ -43,15 +41,33 @@ add_executable(dropshell ${SOURCES})
 
 # Set include directories
 target_include_directories(dropshell PRIVATE 
-    src
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
-    ${CMAKE_CURRENT_BINARY_DIR}/src
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/utils
-    ${xxHash_INCLUDE_DIRS}
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/contrib
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/src>
 )
 
+include(FetchContent)
+FetchContent_Declare(
+  libassert
+  GIT_REPOSITORY https://github.com/jeremy-rifkin/libassert.git
+  GIT_TAG        v2.1.5 # <HASH or TAG>
+)
+FetchContent_MakeAvailable(libassert)
+target_link_libraries(dropshell libassert::assert)
+
+# On windows copy libassert.dll to the same directory as the executable for your_target
+if(WIN32)
+  add_custom_command(
+    TARGET dropshell POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    $<TARGET_FILE:libassert::assert>
+    $<TARGET_FILE_DIR:dropshell>
+  )
+endif()
+
 # Link libraries
 target_link_libraries(dropshell PRIVATE
-    TBB::tbb
-    ${xxHash_LIBRARIES}
 )
 
 # Install targets
@@ -126,14 +142,3 @@ install(CODE "
         message(STATUS \"Command 'ds' not found. Skipping completion symlink.\")
     endif()
 ")
-
-# Create pre-install script to clean old templates
-install(CODE "
-    message(STATUS \"Removing old template files...\")
-    execute_process(COMMAND rm -rf /opt/dropshell/templates)
-")
-
-# Install templates with pre-install script
-install(DIRECTORY templates/
-    DESTINATION /opt/dropshell/templates
-)

build.sh

@@ -74,8 +74,8 @@ fi
 
 # Configure with CMake
 print_status "Configuring with CMake..."
-#cmake .. -DCMAKE_BUILD_TYPE=Debug
+cmake .. -DCMAKE_BUILD_TYPE=Debug
-cmake .. -DCMAKE_BUILD_TYPE=Release
+#cmake .. -DCMAKE_BUILD_TYPE=Release
 
 # Build the project
 print_status "Building project..."

cmake/FindxxHash.cmake

@@ -1,38 +0,0 @@
-# Find xxHash library
-#
-# This sets the following variables:
-#   xxHash_FOUND - True if xxHash was found
-#   xxHash_INCLUDE_DIRS - xxHash include directories
-#   xxHash_LIBRARIES - xxHash libraries
-
-find_path(xxHash_INCLUDE_DIR
-    NAMES xxhash.h
-    PATHS
-        /usr/include
-        /usr/local/include
-        /opt/local/include
-        ${CMAKE_INSTALL_PREFIX}/include
-    PATH_SUFFIXES xxhash
-)
-
-find_library(xxHash_LIBRARY
-    NAMES xxhash libxxhash
-    PATHS
-        /usr/lib
-        /usr/local/lib
-        /opt/local/lib
-        ${CMAKE_INSTALL_PREFIX}/lib
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(xxHash
-    FOUND_VAR xxHash_FOUND
-    REQUIRED_VARS xxHash_LIBRARY xxHash_INCLUDE_DIR
-)
-
-if(xxHash_FOUND)
-    set(xxHash_LIBRARIES ${xxHash_LIBRARY})
-    set(xxHash_INCLUDE_DIRS ${xxHash_INCLUDE_DIR})
-endif()
-
-mark_as_advanced(xxHash_INCLUDE_DIR xxHash_LIBRARY) 

debian/changelog

@@ -1,8 +0,0 @@
-dropshell (1.0.0-1) unstable; urgency=medium
-
-  * Initial release.
-  * Converted from bash script to C++ implementation
-  * Added proper system status monitoring
-  * Improved server management functionality
-
- -- j842 Sun, 21 Apr 2025 12:00:00 +0000 

debian/control

@@ -1,15 +0,0 @@
-Source: dropshell
-Section: utils
-Priority: optional
-Maintainer: j842
-Build-Depends: debhelper (>= 10), cmake, libboost-all-dev
-Standards-Version: 4.5.0
-Homepage: https://github.com/j842/dropshell
-
-Package: dropshell
-Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, libboost-program-options1.74.0, libboost-filesystem1.74.0, libboost-system1.74.0
-Description: A system management tool for server operations
- dropshell is a command-line tool for managing and monitoring servers.
- It provides functionality for checking system status, managing server
- configurations, and performing common administrative tasks. 

debian/copyright

@@ -1,26 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: dropshell
-Source: https://github.com/j842/dropshell
-
-Files: *
-Copyright: 2025 j842 
-License: MIT
-
-License: MIT
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
- .
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
- .
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE. 

debian/rules

@@ -1,14 +0,0 @@
-#!/usr/bin/make -f
-
-%:
-	dh $@
-
-override_dh_auto_configure:
-	dh_auto_configure -- -DCMAKE_BUILD_TYPE=Release
-
-override_dh_auto_install:
-	dh_auto_install
-	# Install bash completion
-	install -D -m 644 src/dropshell-completion.bash debian/dropshell/etc/bash_completion.d/dropshell
-	# Install configuration
-	install -D -m 644 src/dropshell.conf debian/dropshell/etc/dropshell.conf 

docker/Dockerfile.build

@@ -0,0 +1,24 @@
+FROM debian:bullseye AS builder
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    cmake \
+    pkg-config \
+    bash \
+    gcc-aarch64-linux-gnu \
+    g++-aarch64-linux-gnu \
+    binutils-aarch64-linux-gnu \
+    qemu-user-static
+
+WORKDIR /app
+
+COPY . .
+
+COPY --chmod=755 docker/_create_dropshell.sh /scripts/
+
+RUN rm -rf build
+
+ENV CXXFLAGS="-static-libstdc++ -static-libgcc"
+ENV LDFLAGS="-static -pthread -Wl,--whole-archive -lpthread -Wl,--no-whole-archive"
+
+CMD ["/bin/bash","/scripts/_create_dropshell.sh"]

docker/_create_dropshell.sh

@@ -8,5 +8,3 @@ make -j4
 cp /app/build/dropshell /output/
 
 chown $CHOWN_USER:$CHOWN_GROUP /output/dropshell
-
-/app/build/dropshell version

docker/build_builder.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+ROOT_DIR=$(dirname $SCRIPT_DIR)
+
+docker build -t gitea.jde.nz/j/dropshell_builder:latest $ROOT_DIR -f $SCRIPT_DIR/Dockerfile.build
+

docker/compile.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+ROOT_DIR=$(dirname $SCRIPT_DIR)
+
+echo "Building dropshell from $ROOT_DIR"
+
+# Build the builder image
+#docker build -t dropshell_alpine_builder $ROOT_DIR -f $SCRIPT_DIR/Dockerfile.build
+
+rm -rf $SCRIPT_DIR/output
+mkdir -p $SCRIPT_DIR/output
+
+MYUID=$(id -u)  
+MYGID=$(id -g)
+
+# Build for x86_64
+echo "Building for x86_64..."
+docker run --rm -tt --env CHOWN_USER=$MYUID --env CHOWN_GROUP=$MYGID \
+    -v $SCRIPT_DIR/output:/output \
+    -e TARGET_ARCH=x86_64 \
+    -e CC=gcc \
+    -e CXX=g++ \
+    gitea.jde.nz/j/dropshell_builder:latest
+
+mv $SCRIPT_DIR/output/dropshell $SCRIPT_DIR/output/dropshell_x86_64
+
+$SCRIPT_DIR/output/dropshell_x86_64 version
+
+echo "dropshell built in $SCRIPT_DIR/output/dropshell_x86_64"
+
+
+# Build for arm64
+echo "Building for arm64..."
+docker run --rm -tt --env CHOWN_USER=$MYUID --env CHOWN_GROUP=$MYGID \
+    -v $SCRIPT_DIR/output:/output \
+    -e TARGET_ARCH=aarch64 \
+    -e CC=aarch64-linux-gnu-gcc \
+    -e CXX=aarch64-linux-gnu-g++ \
+    gitea.jde.nz/j/dropshell_builder:latest
+
+mv $SCRIPT_DIR/output/dropshell $SCRIPT_DIR/output/dropshell_aarch64
+
+
+echo "dropshell built in $SCRIPT_DIR/output/dropshell_aarch64"
+

docker/dropshell_alpine/Dockerfile.build

@@ -1,21 +0,0 @@
-FROM alpine:latest
-
-RUN apk add --no-cache \
-    build-base \
-    cmake \
-    git \
-    ncurses-dev \
-    pkgconfig \
-    libtbb-dev \
-    xxhash-dev \
-    bash
-
-WORKDIR /app
-
-COPY . .
-
-COPY --chmod=755 docker/dropshell_alpine/_create_dropshell.sh /scripts/
-
-RUN rm -rf build
-
-CMD ["/bin/bash","/scripts/_create_dropshell.sh"]

docker/dropshell_alpine/Dockerfile.run

@@ -1,19 +0,0 @@
-FROM alpine:latest
-
-RUN apk add --no-cache \
-    libtbb \
-    xxhash-dev \
-    bash \
-    nano
-
-WORKDIR /app
-
-COPY docker/dropshell_alpine/output/dropshell /app/dropshell
-RUN /bin/bash -c "mkdir -p /opt/dropshell/templates && mkdir -p /config"
-COPY templates /opt/dropshell/templates
-COPY src/dropshell-completion.bash /etc/bash_completion.d/
-
-RUN /app/dropshell init /config
-RUN /app/dropshell version
-
-ENTRYPOINT ["/app/dropshell"]

docker/dropshell_alpine/build.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-ROOT_DIR=$(dirname $(dirname $SCRIPT_DIR))
-
-echo "Building dropshell from $ROOT_DIR"
-
-docker build -t dropshell_alpine_builder $ROOT_DIR -f $SCRIPT_DIR/Dockerfile.build
-
-rm -rf $SCRIPT_DIR/output
-mkdir -p $SCRIPT_DIR/output
-
-MYUID=$(id -u)  
-MYGID=$(id -g)
-
-docker run -it --env CHOWN_USER=$MYUID --env CHOWN_GROUP=$MYGID -v $SCRIPT_DIR/output:/output dropshell_alpine_builder 
-
-echo "dropshell built in $SCRIPT_DIR/output/dropshell"
-
-docker build -t gitea.jde.nz/j/dropshell_alpine:latest -t dropshell_alpine $ROOT_DIR -f $SCRIPT_DIR/Dockerfile.run
-
-docker run -it dropshell_alpine version

docker/dropshell_alpine/example.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-if [ $# -lt 1 ]; then
-	echo "	Use: example <dir>"
-	exit 0	
-fi
-
-
-(
-cd "$1"
-echo ZG9ja2VyIHJ1biAtdHQgLS1ybSAtdi46L3ggZ2l0ZWEuamRlLm56L2ovZHJvcHNoZWxsX2FscGluZSBoYXNoIC94 | base64 -d | bash
-)
-

docker/dropshell_alpine/publish.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-docker push gitea.jde.nz/j/dropshell_alpine:latest

docker/publish_builder.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker push gitea.jde.nz/j/dropshell_builder:latest

docker/publish_dropshell.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+

install_build_prerequisites.sh

@@ -41,7 +41,7 @@ print_status "Detected OS: $OS $VER"
 case $OS in
     "Ubuntu"|"Debian GNU/Linux")
         # Common packages for both Ubuntu and Debian
-        PACKAGES="cmake make g++ devscripts debhelper libtbb-dev libxxhash-dev"
+        PACKAGES="cmake make g++ devscripts debhelper"
         ;;
     *)
         print_error "Unsupported distribution: $OS"
@@ -54,6 +54,10 @@ is_package_installed() {
     dpkg -l "$1" 2>/dev/null | grep -q "^ii"
 }
 
+# Update package lists
+print_status "Updating package lists..."
+apt-get update
+
 # Install missing packages
 print_status "Checking and installing required packages..."
 for pkg in $PACKAGES; do
@@ -69,10 +73,6 @@ for pkg in $PACKAGES; do
     fi
 done
 
-# Update package lists
-print_status "Updating package lists..."
-apt-get update
-
 # Verify all required tools are installed
 print_status "Verifying installation..."
 for tool in cmake make g++; do
@@ -82,17 +82,5 @@ for tool in cmake make g++; do
     fi
 done
 
-# Check TBB installation
-if [ ! -d "/usr/include/tbb" ]; then
-    print_error "TBB headers not found"
-    exit 1
-fi
-
-# # Check Boost installation
-# if [ ! -d "/usr/include/boost" ]; then
-#     print_error "Boost headers not found"
-#     exit 1
-# fi
-
 print_status "All dependencies installed successfully!"
 print_status "You can now run ./build.sh to build the project" 

remote_host_autosetup/host_autosetup.sh

@@ -0,0 +1,107 @@
+# can you make this script run in bash, but fall back to sh if bash is not installed?
+
+# check if we are running as root
+if [ "$(id -u)" -ne 0 ]; then
+    echo "Please run the script with sudo privileges or as root."
+    exit 1
+fi
+
+# install bash if not already installed.
+if ! command -v bash > /dev/null 2>&1; then
+    echo "bash is not installed"
+    apt update
+    apt install -y bash
+fi
+
+# re-exec this script in bash if not already running in bash
+if [ -z "$BASH_VERSION" ]; then
+    echo "Re-executing script in Bash..."
+    exec /bin/bash "$0" "$@"
+fi
+
+if [ -n "$BASH_VERSION" ]; then
+    echo "Running in Bash $BASH_VERSION"
+else
+    echo "Not running in Bash"
+    exit 1
+fi
+
+# check if curl, wget, bash installed, and install if not
+PREREQUISITES=("curl" "wget" "jq")
+
+# check if all prerequisites are installed
+ALLINSTALLED=true
+for prerequisite in "${PREREQUISITES[@]}"; do
+    if ! command -v "${prerequisite}" &> /dev/null; then
+        echo "Prerequisite: ${prerequisite} is not installed."
+        ALLINSTALLED=false
+    fi
+done
+
+if [ "$ALLINSTALLED" = false ]; then
+    echo "Installing prerequisites..."
+    apt update
+    apt install -y "${PREREQUISITES[@]}"
+fi
+
+#--------------------------------
+
+# check docker installation
+if ! command -v docker &> /dev/null; then
+    echo "Docker is not installed."
+    echo "Installing docker..."
+    curl -fsSL https://get.docker.com -o get-docker.sh
+    sh get-docker.sh
+    rm get-docker.sh
+fi
+
+# check dropshell user exists
+if ! id "dropshell" &> /dev/null; then
+    echo "Dropshell user does not exist."
+    echo "Creating dropshell user..."
+    useradd -m dropshell
+fi
+
+# add dropshell user to docker group
+# check if already in docker group
+if ! id "dropshell" | grep -q "docker" &> /dev/null; then
+    echo "Adding dropshell user to docker group..."
+    usermod -aG docker dropshell
+fi
+
+# check .ssh/authorized_keys file exists
+if [ ! -f "/home/dropshell/.ssh/authorized_keys" ]; then
+    echo "Creating .ssh/authorized_keys file..."
+    mkdir -p /home/dropshell/.ssh
+    cp /root/.ssh/authorized_keys /home/dropshell/.ssh/authorized_keys
+    chown -R dropshell:dropshell /home/dropshell/.ssh
+    chmod 600 /home/dropshell/.ssh/authorized_keys
+fi
+
+# also check known_hosts file exists
+if [ ! -f "/home/dropshell/.ssh/known_hosts" ]; then
+    echo "Creating .ssh/known_hosts file..."
+    mkdir -p /home/dropshell/.ssh
+    touch /home/dropshell/.ssh/known_hosts
+fi
+
+# ensure default shell for dropshell user is bash
+chsh -s /bin/bash dropshell
+
+#--------------------------------
+
+# download dropshell
+
+# determine if x86_64 or arm64
+ARCH=$(uname -m)
+
+# check is aarch64 or x86_64 and error if neither
+if [ "$ARCH" != "aarch64" ] && [ "$ARCH" != "x86_64" ]; then
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+fi
+
+echo "Installation complete."
+
+#--------------------------------
+

src/autocomplete.cpp

@@ -4,17 +4,17 @@
 #include "templates.hpp"
 #include "services.hpp"
 #include "servers.hpp"
-#include "utils/assert.hpp"
 
+#include <libassert/assert.hpp>
 #include <algorithm>
 #include <iostream>
 
-void dropshell::autocomplete(const std::vector<std::string> &args)
+bool dropshell::autocomplete(const std::vector<std::string> &args)
 {
     if (args.size() < 3) // dropshell autocomplete ???
     {
         autocomplete_list_commands();
-        return;
+        return true;
     }
 
     ASSERT(args.size() >= 3);
@@ -22,19 +22,28 @@ void dropshell::autocomplete(const std::vector<std::string> &args)
 
     // std::cout<<" cmd = ["<<cmd<<"]"<<std::endl;
 
+    if (cmd=="hash")
+    {   // output files and folders in the current directory, one per line.
+        std::filesystem::directory_iterator dir_iter(std::filesystem::current_path());
+        for (const auto& entry : dir_iter) 
+            std::cout << entry.path().filename().string() << std::endl;        
+        return true;
+    }
+
+
     std::string noargcmds[] = {"templates","autocomplete_list_servers","autocomplete_list_services","autocomplete_list_commands"};
     if (std::find(std::begin(noargcmds), std::end(noargcmds), cmd) != std::end(noargcmds))
-        return;
+        return true;
 
     if (!dropshell::gConfig().is_config_set())
-        return; // can't help without working config.
+        return false; // can't help without working config.
 
     if (args.size()==3) // we have the command but nothing else.  dropshell autocomplete command <server>
     {
         auto servers = dropshell::get_configured_servers();
         for (const auto& server : servers) 
             std::cout << server.name << std::endl;
-        return;
+        return true;
     }
 
     if (args.size()==4) // we have the command and the server.  dropshell autocomplete command server <service>
@@ -43,17 +52,16 @@ void dropshell::autocomplete(const std::vector<std::string> &args)
 
         if (cmd=="create-service")
         { // create-service <server> <template> <service>
-            std::vector<template_info> templates;
+            auto templates = dropshell::gTemplateManager().get_template_list();
-            get_templates(templates);
             for (const auto& t : templates)
-                std::cout << t.template_name << std::endl;
+                std::cout << t << std::endl;
-            return;
+            return true;
         }
 
         auto services = dropshell::get_server_services_info(server);
         for (const auto& service : services) 
             std::cout << service.service_name << std::endl;
-        return;
+        return true;
     }
 
     if (args.size()==5) // we have the command and the server and the service.  dropshell autocomplete command server service_name <command?>
@@ -65,32 +73,33 @@ void dropshell::autocomplete(const std::vector<std::string> &args)
             std::set<std::string> backups = dropshell::list_backups(server_name, service_name);
             for (auto backup : backups) 
                 std::cout << backup << std::endl;
-            return;
+            return true;
         }
 
-        return; // no more autocompletion possible - don't know what the argument is for.
+        return false; // no more autocompletion possible - don't know what the argument is for.
     }
 
     // args>5 - no more autocompletion possible - don't know what the argument is for.
-    return; // catch-all.
+    return false; // catch-all.
 }
 
-void dropshell::autocomplete_list_commands()
+bool dropshell::autocomplete_list_commands()
 {
     std::set<std::string> commands;
     dropshell::get_all_used_commands(commands);
 
 // add in commmands hard-coded and handled in main
     commands.merge(std::set<std::string>{ 
-        "help","init" // these are always available.
+        "help","edit" // these are always available.
     });
     if (dropshell::gConfig().is_config_set()) 
         commands.merge(std::set<std::string>{
-            "server","templates","create-service","create-template","create-server","edit","ssh",
+            "server","templates","create-service","create-template","create-server","ssh",
             "list" // only if we have a config.
         });
     for (const auto& command : commands) {
         std::cout << command << std::endl;
     }
+    return true;
 }
 

src/autocomplete.hpp

@@ -7,9 +7,9 @@
 
 namespace dropshell {
 
-    void autocomplete(const std::vector<std::string> &args);
+    bool autocomplete(const std::vector<std::string> &args);
 
-    void autocomplete_list_commands();
+    bool autocomplete_list_commands();
 
 } // namespace dropshell
 

src/config.cpp

@@ -2,9 +2,8 @@
 #include <iostream>
 #include <fstream>
 #include "config.hpp"
-#include "utils/envmanager.hpp"
 #include "utils/utils.hpp"
-
+#include "utils/json.hpp"
 #include <filesystem>
 
 namespace dropshell {
@@ -16,103 +15,146 @@ config & gConfig() {
 }
 
 
-config::config() {
+config::config() : mIsConfigSet(false) {
 }
+
 config::~config() {
 }
 
-bool config::load_config() {
+bool config::load_config() { // load json config file.
-    std::string config_path = localfile::dropshell_env();
+    std::string config_path = localfile::dropshell_json();
     if (config_path.empty() || !std::filesystem::exists(config_path))
         return false;
 
-    envmanager config_env(config_path);
+    std::ifstream config_file(config_path);
-    if (!config_env.load())
+    if (!config_file.is_open())
         return false;
 
-    std::string directories = config_env.get_variable_substituted("local.config.directories");
+    try {
-    if (directories.empty())
+        mConfig = nlohmann::json::parse(config_file);
+    }
+    catch (nlohmann::json::parse_error& ex)
+    {
+        std::cerr << "Error: Failed to parse config file: " << ex.what() << std::endl;
         return false;
+    }
 
-    // Split the directories string into a vector of strings
+    mIsConfigSet = true;
-    mLocalConfigPaths = string2multi(directories);
-
-    mLocalBackupPath = config_env.get_variable_substituted("local.backup.directory");
-
-    // legacy config file conversion
-    if (mLocalBackupPath.empty() && mLocalConfigPaths.size()>0)
-        mLocalBackupPath = mLocalConfigPaths[0] + "/backups";
-
-    //std::cout << "Local config path: " << mLocalConfigPath << std::endl;
     return true;
 }
 
-void config::save_config()
+bool config::save_config(bool create_aux_directories)
 {
-    if (mLocalConfigPaths.empty())
+    std::string config_path = localfile::dropshell_json();
+    if (config_path.empty())
+        return false;
+
+    std::filesystem::create_directories(get_parent(config_path));
+
+    std::ofstream config_file(config_path);
+    if (!config_file.is_open())
+        return false;
+
+    if (!mIsConfigSet)
     {
-        std::cerr << "Warning: Unable to save configuration file, as DropShell has not been initialised."<< std::endl;
+        std::string homedir = localpath::current_user_home();
-        std::cerr << "Please run 'dropshell init <path>' to initialise DropShell." << std::endl;
+        std::string dropshell_base = homedir + "/.dropshell";
-        return;
+        mConfig["tempfiles"] = dropshell_base + "/tmp";
+        mConfig["backups"] = dropshell_base + "/backups";
+        mConfig["template_cache"] = dropshell_base + "/template_cache";
+        mConfig["template_registry_URLs"] = {
+            "https://templates.dropshell.app"
+        };
+        mConfig["template_local_paths"] = {
+            dropshell_base + "/local_templates"
+        };
+        mConfig["server_definition_paths"] = {
+            dropshell_base + "/servers"
+        };
+        mConfig["template_upload_registry_url"] = "https://templates.dropshell.app"; 
+        mConfig["template_upload_registry_token"] = "SECRETTOKEN";
     }   
 
-    std::string parent_path = dropshell::get_parent(localfile::dropshell_env());
+    config_file << mConfig.dump(4);
-    std::filesystem::create_directories(parent_path);
+    config_file.close();
 
-    std::string config_path = localfile::dropshell_env();
+    if (create_aux_directories) {
-    envmanager config_env(config_path);
+        std::vector<std::filesystem::path> paths = {
+            get_local_template_cache_path(),
+            get_local_backup_path(),
+            get_local_tempfiles_path()
+        };
+        for (auto & p : get_local_server_definition_paths()) 
+            paths.push_back(p);
 
-    config_env.set_variable("local.config.directories", multi2string(mLocalConfigPaths));
+        for (auto & p : paths)
-    config_env.set_variable("local.backup.directory", mLocalBackupPath);
+            if (!std::filesystem::exists(p))
-    config_env.save();
+            {
+                std::cout << "Creating directory: " << p << std::endl;
+                std::filesystem::create_directories(p);
             }
+    }
+
+    return true;
+}
+  
 
 bool config::is_config_set() const
 {   
-    return !mLocalConfigPaths.empty() && !mLocalBackupPath.empty();
+    return mIsConfigSet;
 }
 
+std::string config::get_local_tempfiles_path() {
+    return mConfig["tempfiles"];
+}
 
-const std::vector<std::string> & config::get_local_config_directories() const
+std::string config::get_local_backup_path() {
+    return mConfig["backups"];
+}
+
+std::string config::get_local_template_cache_path() {
+    return mConfig["template_cache"];
+}
+
+std::vector<std::string> config::get_template_registry_urls() {
+    nlohmann::json template_registry_urls = mConfig["template_registry_URLs"];
+    std::vector<std::string> urls;
+    for (auto &url : template_registry_urls) {
+        urls.push_back(url);
+    }
+    return urls;
+}
+
+std::vector<std::string> config::get_template_local_paths()
 {
-    return mLocalConfigPaths;
+    nlohmann::json template_local_paths = mConfig["template_local_paths"];
+    std::vector<std::string> paths;
+    for (auto &path : template_local_paths) {
+        if (path.is_string() && !path.empty())
+            paths.push_back(path);
+    }
+    return paths;
 }
 
-bool config::add_local_config_directory(const std::string &path)
+std::vector<std::string> config::get_local_server_definition_paths() {
-{
+    nlohmann::json server_definition_paths = mConfig["server_definition_paths"];
-    if (path.empty())
-        return false;
 
-    // Convert to canonical path, using std::filesystem
+    std::vector<std::string> paths;
-    std::filesystem::path abs_path = std::filesystem::canonical(path);
+    for (auto &path : server_definition_paths) {
-    
+        if (path.is_string() && !path.empty())
-    // The directory must exist
+            paths.push_back(path);
-    if (!std::filesystem::exists(abs_path)) {
+        else
-        std::cerr << "Error: The local config directory does not exist: " << abs_path.string() << std::endl;
+            std::cerr << "Warning: Invalid server definition path: " << path << std::endl;
-        return false;
+    }
+    return paths;
 }
 
-    // Add to config paths if not already there
+std::string config::get_template_upload_registry_url() {
-    for (auto &p : mLocalConfigPaths)
+    return mConfig["template_upload_registry_url"];
-    { // robustly compare the two paths.
-        if (p == abs_path.string())
-        {
-            std::cerr << "Warning: The local config directory is already registered: " << abs_path.string() << std::endl;
-            std::cerr << "No changes made to the DropShell configuration." << std::endl;
-            return false;
-        }
-    }
-    mLocalConfigPaths.push_back(abs_path.string());
-
-    if (mLocalBackupPath.empty())
-            mLocalBackupPath = abs_path.string() + "/backups";
-
-    return true;
 }
 
-const std::string &config::get_local_backup_path() const
+std::string config::get_template_upload_registry_token() {
-{
+    return mConfig["template_upload_registry_token"];
-    return mLocalBackupPath;
 }
 
 } // namespace dropshell 

src/config.hpp

@@ -2,6 +2,7 @@
 
 #include <string>
 #include <vector>
+#include "utils/json.hpp"
 
 namespace dropshell {
 
@@ -9,19 +10,27 @@ class config {
     public:
         config();
         ~config();
+
         bool load_config();
-        void save_config();
+        bool save_config(bool create_aux_directories);
 
         bool is_config_set() const;
 
-        const std::vector<std::string> & get_local_config_directories() const;
-        bool add_local_config_directory(const std::string& path);
 
-        const std::string & get_local_backup_path() const;
+        std::string get_local_tempfiles_path();
+        std::string get_local_backup_path();
+        std::string get_local_template_cache_path();
+
+        std::vector<std::string> get_template_registry_urls();
+        std::vector<std::string> get_template_local_paths();
+        std::vector<std::string> get_local_server_definition_paths();
+
+        std::string get_template_upload_registry_url();
+        std::string get_template_upload_registry_token();
 
     private:
-        std::vector<std::string> mLocalConfigPaths;
+        nlohmann::json mConfig;
-        std::string mLocalBackupPath;
+        bool mIsConfigSet;
 };
 
 

src/main.cpp

@@ -6,9 +6,7 @@
 #include "utils/directories.hpp"
 #include "templates.hpp"
 #include "utils/utils.hpp"
-#include "utils/readmes.hpp"
 #include "autocomplete.hpp"
-#include "main_commands.hpp"
 #include "utils/hash.hpp"
 
 #include <filesystem>
@@ -17,7 +15,7 @@
 #include <vector>
 #include <iomanip>
 #include <chrono>
-
+#include <libassert/assert.hpp>
 
 namespace dropshell {
 
@@ -26,7 +24,7 @@ extern const std::string RELEASE_DATE;
 extern const std::string AUTHOR;
 extern const std::string LICENSE;
 
-void print_help() {
+bool print_help() {
     std::cout << std::endl;
     maketitle("DropShell version " + VERSION);
     std::cout << std::endl;
@@ -34,7 +32,7 @@ void print_help() {
     std::cout << std::endl;
     std::cout << "dropshell ..." << std::endl;
     std::cout << "  help           Show this help message" << std::endl;
-    std::cout << "  init DIR       Add DIR as a local server config directory (can add several)" << std::endl;
+    std::cout << "  edit           Edit the configuration of dropshell" << std::endl;
 
     if (gConfig().is_config_set()) {
     std::cout << "  server NAME    Show details for specific server" << std::endl;
@@ -42,14 +40,17 @@ void print_help() {
     std::cout << std::endl;
     std::cout << std::endl;
     std::cout << "Service commands: (if no service is specified, all services for the server are affected)" << std::endl;
-    std::cout << "  install SERVER [SERVICE]     Install/reinstall/update service(s). Non-destructive." << std::endl;
     std::cout << "  list [SERVER] [SERVICE]      List status/details of all servers/server/service." << std::endl;
     std::cout << "  edit [SERVER] [SERVICE]      Edit the configuration of dropshell/server/service." << std::endl;
-    std::cout << "  COMMAND SERVER [SERVICE]     Run a command on service(s)." << std::endl;
     std::cout << std::endl;
-    std::cout << "Standard commands: install, uninstall, backup, restore, start, stop" << std::endl;
+    std::cout << "  install SERVER [SERVICE]     Install/reinstall/update service(s). Safe/non-destructive." << std::endl;
+    std::cout << "  uninstall SERVER [SERVICE]   Uninstalls the service on the remote server. Leaves data intact." << std::endl;
+    std::cout << "  nuke SERVER SERVICE          Nuke the service on the remote server, deleting all remote data." << std::endl;
     std::cout << std::endl;
-    std::cout << "  ssh     SERVER [SERVICE]     Launch an interactive shell on a server or service" << std::endl;
+    std::cout << "  COMMAND SERVER [SERVICE]     Run a command on service(s), e.g." << std::endl;
+    std::cout << "                                  backup, restore, start, stop, logs" << std::endl;
+    std::cout << std::endl;
+    std::cout << "  ssh     SERVER SERVICE       Launch an interactive shell on a server or service" << std::endl;
     std::cout << std::endl;
     std::cout << "Creation commands: (apply to the first local config directory)"<<std::endl;
     std::cout << "  create-template TEMPLATE" << std::endl;
@@ -61,6 +62,7 @@ void print_help() {
         std::cout << std::endl;
         std::cout << "Other commands available once initialised." << std::endl;
     }
+    return true;
 }
 
 
@@ -69,17 +71,22 @@ int die(const std::string & msg) {
     return 1;
 }
 
-bool parseargs(std::string arg2, std::string arg3, std::string & server_name, std::vector<ServiceInfo>& servicelist)
+struct ServerAndServices {
+    std::string server_name;
+    std::vector<LocalServiceInfo> servicelist;
+};
+
+bool getCLIServices(const std::string & arg2, const std::string & arg3, 
+    ServerAndServices & server_and_services)
 {
     if (arg2.empty()) return false;
-    server_name = arg2;
+    server_and_services.server_name = arg2;
 
     if (arg3.empty()) {
-        servicelist = get_server_services_info(server_name);
+        server_and_services.servicelist = get_server_services_info(arg2);
     } else {
-        servicelist.push_back(get_service_info(server_name, arg3));
+        server_and_services.servicelist.push_back(get_service_info(arg2, arg3));
     }
-
     return true;
 }
 
@@ -97,58 +104,66 @@ void printversion() {
 }
 
 #define HAPPYEXIT(CMD, RUNCMD) {if (safearg(argc,argv,1) == CMD) {RUNCMD; return 0;}}
+#define BOOLEXIT(CMD, RUNCMD) {if (safearg(argc,argv,1) == CMD) {return (RUNCMD) ? 0 : 1;}}
 
 int main(int argc, char* argv[]) {
     HAPPYEXIT("hash", hash_demo_raw(safearg(argc,argv,2)))
-    HAPPYEXIT("makesafecmd", std::cout<<makesafecmd(safearg(argc,argv,2))<<std::endl)
     HAPPYEXIT("version", printversion())
-    ASSERT_MSG(safearg(argc,argv,1) != "assert", "Hello! Here is an assert.");
+    BOOLEXIT("test-template", gTemplateManager().test_template(safearg(argc,argv,2)))
+    ASSERT(safearg(argc,argv,1) != "assert", "Hello! Here is an assert.");
 
     try {
-        // silently attempt to load the config file.
+        // silently attempt to load the config file and templates.
         gConfig().load_config();
+        if (gConfig().is_config_set()) 
+            gTemplateManager().load_sources();
+
+        if (argc < 2) 
+            return print_help() ? 0 : 1;
 
-        if (argc < 2) {
-            print_help();
-            return 0;
-        }
         std::string cmd = argv[1];
+
+        if (cmd == "autocomplete") {
             std::vector<std::string> argvec;
             for (int i=0; i<argc; i++) 
                 argvec.push_back(argv[i]);
-
+            return autocomplete(argvec) ? 0 : 1;
-
-        if (cmd == "autocomplete") {
-            autocomplete(argvec);
-            return 0;
         }
 
-        if (cmd == "init") {
+        if (cmd == "help" || cmd == "-h" || cmd == "--help" || cmd== "h" || cmd=="halp") 
-            return main_commands::init(argvec);
+            return print_help() ? 0 : 1;
-        }
-
-        if (cmd == "help" || cmd == "-h" || cmd == "--help" || cmd== "h" || cmd=="halp") {
-            print_help();
-            return 0;
-        }
 
         if (cmd == "edit" && argc < 3) {
-            std::string config_file = localfile::dropshell_env();
+            if (!gConfig().is_config_set())
-            std::filesystem::create_directories(  get_parent(config_file) );
+               gConfig().save_config(false);
-            edit_file(config_file, "Please ensure any directories you have introduced in the config file exist.");
+
+            std::string config_file = localfile::dropshell_json();
+            if (!service_runner::edit_file(config_file) || !std::filesystem::exists(config_file))
+                return die("Error: Failed to edit config file.");
+
+            gConfig().load_config();
+            if (!gConfig().is_config_set())
+                return die("Error: Failed to load and parse edited config file.");
+
+            gConfig().save_config(true);
+            std::cout << "Successfully edited config file at " << config_file << std::endl;
             return 0;
         }
 
         // ------------------------------------------------------------
         // from here we require the config file to be loaded.
         if (!gConfig().is_config_set()) 
-            return die("Please run 'dropshell init <path>' to initialise the user directory and create a configuration file.");
+            return die("Please run 'dropshell edit' to set up the dropshell configuration.");
 
-        const std::vector<std::string> & local_config_directories = gConfig().get_local_config_directories();
+        const std::vector<std::string> & server_definition_paths = gConfig().get_local_server_definition_paths();
-        std::cout << "Config directories: ";
+        if (server_definition_paths.size()>1) { // only show if there are multiple.
-        for (auto & dir : local_config_directories) 
+            std::cout << "Server definition paths: ";
+            for (auto & dir : server_definition_paths) 
                 std::cout << "["<< dir << "] ";
             std::cout << std::endl;
+        }
+        if (gTemplateManager().is_loaded() && gTemplateManager().get_source_count() > 0)
+            gTemplateManager().print_sources();
 
         if (cmd == "server" || cmd == "servers" || cmd == "list" || cmd == "view") 
             switch (argc)
@@ -167,77 +182,72 @@ int main(int argc, char* argv[]) {
             }
 
         if (cmd == "templates") {
-            list_templates();
+            gTemplateManager().list_templates();
             return 0;
         }
 
         if (cmd == "create-template") {
             if (argc < 3) return die("Error: create-template requires a template name");
-            create_template(argv[2]);
+            return (gTemplateManager().create_template(argv[2])) ? 0 : 1;
-            return 0;
         }
 
         if (cmd == "create-server") {
             if (argc < 3) return die("Error: create-server requires a server name");
-            create_server(argv[2]);
+            return (create_server(argv[2])) ? 0 : 1;
-            return 0;
         }
 
         if (cmd == "create-service") {
             if (argc < 5) return die("Error: not enough arguments.\ndropshell create-service server template service");
-            create_service(argv[2], argv[3], argv[4]);
+            return (create_service(argv[2], argv[3], argv[4])) ? 0 : 1;
-            return 0;
         }
 
          if (cmd == "ssh" && argc < 4) {
             if (argc < 3) return die("Error: ssh requires a server name and optionally service name");
-            interactive_ssh(argv[2], "bash");
+            service_runner::interactive_ssh(argv[2], "bash");
             return 0;
         }
 
         if (cmd == "edit" && argc < 4) {
-            ASSERT_MSG(argc>=3, "Error: logic error!");
+            ASSERT(argc>=3, "Error: logic error!");
-            edit_server(safearg(argc,argv,2));
+            service_runner::edit_server(safearg(argc,argv,2));
             return 0;
         }
 
-        if (cmd == "backup" || cmd=="backups") {
-            if (argc < 4) return die("Error: backup requires a target server and target service to back up");
-            return main_commands::backup(argvec);
-        }
-
-        if (cmd == "restore") {
-            if (argc < 4) return die("Error: restore requires a target server, target service the backup file to restore");
-            return main_commands::restore(argvec);
-        }
-
         // handle running a command. 
         std::set<std::string> commands;
         get_all_used_commands(commands);
-        commands.merge(std::set<std::string>{"ssh","edit","_allservicesstatus"}); // handled by service_runner, but not in template_shell_commands.
+        commands.merge(std::set<std::string>{"ssh","edit","_allservicesstatus","fullnuke"}); // handled by service_runner, but not in template_shell_commands.
-        for (const auto& command : commands) {
+
-            if (cmd == command) {
+        if (commands.count(cmd)) {
-                std::string server_name;
+            std::set<std::string> safe_commands = {"nuke", "fullnuke"};
-                std::vector<ServiceInfo> servicelist;
+            if (safe_commands.count(cmd) && argc < 4) 
-                if (!parseargs(safearg(argc, argv, 2), safearg(argc, argv, 3), server_name, servicelist)) {
+                return die("Error: "+cmd+" requires a server name and service name. For safety, can't run on all services.");
-                    std::cerr << "Error: " << command << " command requires server name and optionally service name" << std::endl;
+
-                    return 1;
+            // get all the services to run the command on.
+            ServerAndServices server_and_services;
+            if (!getCLIServices(safearg(argc, argv, 2), safearg(argc, argv, 3), server_and_services)) 
+                return die("Error: "+cmd+" command requires server name and optionally service name");
+
+            // run the command on each service.
+            for (const auto& service_info : server_and_services.servicelist) {
+                if (!SIvalid(service_info))
+                    std::cerr<<"Error: Unable to get service information."<<std::endl;
+                else {
+                    service_runner runner(server_and_services.server_name, service_info.service_name);
+                    if (!runner.isValid()) 
+                        return die("Error: Failed to initialize service");
+                    
+                    std::vector<std::string> additional_args;
+                    for (int i=4; i<argc; i++)
+                        additional_args.push_back(argv[i]);
+                    if (!runner.run_command(cmd, additional_args)) 
+                        return die(cmd+" failed on service "+service_info.service_name);
+                }
             }
 
-                for (const auto& service_info : servicelist) {
+            // success!
-                    service_runner runner(server_name, service_info.service_name);
-                    if (!runner.isValid()) {
-                        std::cerr << "Error: Failed to initialize service" << std::endl;
-                        return 1;
-                    }
-                    if (!runner.run_command(command)) {
-                        std::cerr << command +" failed." << std::endl;
-                        return 1;
-                    }
-                }
             return 0;
         }
-        }
 
         // Unknown command
         std::cerr << "Error: Unknown command '" << cmd << "'" << std::endl;

src/main_commands.cpp

@@ -1,280 +0,0 @@
-#include "main_commands.hpp"
-
-#include "utils/directories.hpp"
-#include "utils/utils.hpp"
-#include "utils/readmes.hpp"
-#include "service_runner.hpp"
-#include "config.hpp"
-#include "templates.hpp"
-#include "server_env_manager.hpp"
-
-#include <iostream>
-#include <filesystem>
-#include <algorithm>
-
-namespace dropshell {
-
-namespace main_commands {
-
-
-static const std::string magic_string = "-_-";
-
-
-int init(const std::vector<std::string> &args)
-{
-    std::string lcd;
-
-    if (args.size() < 3) {
-        std::cerr << "Error: init command requires a directory argument" << std::endl;
-        return 1;
-    }
-    try {
-        if (!gConfig().add_local_config_directory(args[2])) 
-            return 1; // error already reported
-
-        gConfig().save_config();
-        std::cout << "Config directory added: " << gConfig().get_local_config_directories().back() << std::endl;
-        dropshell::create_readme_local_config_dir(gConfig().get_local_config_directories().back());
-        
-        if (gConfig().get_local_config_directories().size() ==1)
-            std::cout << "DropShell is now initialised and you can add a server with 'dropshell create-server <server-name>'" << std::endl;
-        else
-        {
-            std::cout << "DropShell will now use all of the following directories for configuration:" << std::endl;
-            for (const auto& dir : gConfig().get_local_config_directories()) {
-                std::cout << "  " << dir << std::endl;
-            }
-            std::cout << "You can edit the config file manually with:   dropshell edit" << std::endl;
-        }
-    } catch (const std::exception& e) {
-        std::cerr << "Error in init: " << e.what() << std::endl;
-        return 1;
-    }
-    return 0;
-}
-
-int restore(const std::vector<std::string> &args, bool silent)
-{
-    if (args.size() < 4) {
-        std::cerr << "Error: not enough arguments.  dropshell restore <server> <service> <backup-file>" << std::endl;
-        return 1;
-    }
-
-    std::string server_name = args[2];
-    std::string service_name = args[3];
-    std::string backup_file = args[4];
-
-    ServiceInfo service_info = get_service_info(server_name, service_name);
-    if (service_info.local_service_path.empty()) {
-        std::cerr << "Error: Service not found" << std::endl;
-        return 1;
-    }
-
-    server_env_manager env(server_name);
-    if (!env.is_valid()) {
-        std::cerr << "Error: Invalid server environment" << std::endl;
-        return 1;
-    }
-
-    std::string local_backups_dir = localpath::backups_path();
-    std::string local_backup_file_path = (std::filesystem::path(local_backups_dir) / backup_file).string();
-
-    if (! std::filesystem::exists(local_backup_file_path)) {
-        std::cerr << "Error: Backup file not found at " << local_backup_file_path << std::endl;
-        return 1;
-    }
-
-    // split the backup filename into parts based on the magic string
-    std::vector<std::string> parts = dropshell::split(backup_file, magic_string);
-    if (parts.size() != 4) {
-        std::cerr << "Error: Backup file format is incompatible, - in one of the names?" << std::endl;
-        return 1;
-    }
-
-    std::string backup_server_name = parts[0];
-    std::string backup_template_name = parts[1];
-    std::string backup_service_name = parts[2];
-    std::string backup_datetime = parts[3];
-
-    if (backup_template_name != service_info.template_name) {
-        std::cerr << "Error: Backup template does not match service template. Can't restore." << std::endl;
-        return 1;
-    }
-
-    std::string nicedate = std::string(backup_datetime).substr(0, 10);
-
-    std::cout << "Restoring " << nicedate << " backup of " << backup_template_name << " taken from "<<backup_server_name<<", onto "<<server_name<<std::endl;
-    std::cout << std::endl;
-    std::cout << "*** ALL DATA FOR "<<server_name<<"/"<<service_name<<" WILL BE OVERWRITTEN! ***"<<std::endl;
-    std::cout << std::endl;
-    std::cout << "Are you sure you want to continue? (y/n)" << std::endl;
-    char confirm;
-    std::cin >> confirm;
-    if (confirm != 'y') {
-        std::cout << "Restore cancelled." << std::endl;
-        return 1;
-    }
-
-    // run the restore script
-    std::cout << "OK, here goes..." << std::endl;
-
-    { // backup existing service
-        std::cout << "1) Backing up existing service... " << std::flush;
-        std::vector<std::string> backup_args = {"dropshell","backup",server_name, service_name};
-        if (!backup(backup_args,true)) // silent=true
-        {
-            std::cerr << std::endl;
-            std::cerr << "Error: Backup failed, restore aborted." << std::endl;
-            std::cerr << "You can try using   dropshell install "<<server_name<<" "<<service_name<<"   to install the service afresh." << std::endl;
-            std::cerr << "Otherwise, stop the service, create and initialise a new one, then restore to that." << std::endl;
-            return 1;
-        }
-        std::cout << "Backup complete." << std::endl;
-    }
-
-    { // restore service from backup
-        std::cout << "2) Restoring service from backup..." << std::endl;
-        std::string remote_backups_dir = remotepath::backups(server_name);
-        std::string remote_backup_file_path = remote_backups_dir + "/" + backup_file;
-
-        // Copy backup file from local to server
-        std::string scp_cmd = "scp -P " + env.get_SSH_PORT() + " " + quote(local_backup_file_path) + " " + env.get_SSH_USER() + "@" + env.get_SSH_HOST() + ":" + quote(remote_backup_file_path) + (silent ? " > /dev/null 2>&1" : "");
-        if (!env.execute_local_command(scp_cmd)) {
-            std::cerr << "Failed to copy backup file from server" << std::endl;
-            return false;
-        }
-        env.run_remote_template_command(service_name, "restore", {remote_backup_file_path}, silent);
-    }
-
-    // healthcheck the service
-    std::cout << "3) Healthchecking service..." << std::endl;
-    std::string green_tick = "\033[32m✓\033[0m";
-    std::string red_cross = "\033[31m✗\033[0m";    
-    bool healthy= (env.run_remote_template_command(service_name, "status", {}, silent));
-    if (!silent)
-        std::cout << (healthy ? green_tick : red_cross) << " Service is " << (healthy ? "healthy" : "NOT healthy") << std::endl;
-
-    return 0;
-}
-
-bool name_breaks_backups(std::string name)
-{
-    // if name contains -_-, return true
-    return name.find("-_-") != std::string::npos;
-}
-
-
-        // backup the service over ssh, using the credentials from server.env (via server_env.hpp)
-        // 1. run backup.sh on the server
-        // 2. create a backup file with format server-service-datetime.tgz
-        // 3. store it in the server's DROPSHELL_DIR/backups folder
-        // 4. copy it to the local user_dir/backups folder
-
-// ------------------------------------------------------------------------------------------------
-// Backup the service.
-// ------------------------------------------------------------------------------------------------
-int backup(const std::vector<std::string> & args, bool silent) {
-    if (args.size() < 4) {
-        std::cerr << "Error: backup command requires a server name and service name" << std::endl;
-        return 1;
-    }
-
-    std::string server_name = args[2];
-    std::string service_name = args[3];
-
-    ServiceInfo service_info = get_service_info(server_name, service_name);
-    if (service_info.local_service_path.empty()) {
-        std::cerr << "Error: Service not found" << std::endl;
-        return 1;
-    }
-
-    server_env_manager env(server_name);
-    if (!env.is_valid()) {
-        std::cerr << "Error: Invalid server environment" << std::endl;
-        return 1;
-    }
-
-    const std::string command = "backup";
-
-    if (!template_command_exists(service_info.template_name, command)) {
-        std::cout << "No backup script for " << service_info.template_name << std::endl;
-        return true; // nothing to back up.
-    }
-    
-    // Check if basic installed stuff is in place.
-    std::string remote_service_template_path = remotepath::service_template(server_name, service_name);
-    std::string remote_command_script_file = remote_service_template_path + "/" + command + ".sh";
-    std::string remote_service_config_path = remotepath::service_config(server_name, service_name);
-    if (!env.check_remote_items_exist({
-        remotepath::service(server_name, service_name),
-        remote_command_script_file,
-        remotefile::service_env(server_name, service_name)})
-    )
-    {
-        std::cerr << "Error: Required service directories not found on remote server" << std::endl;
-        std::cerr << "Is the service installed?" << std::endl;
-        return false;
-    }
-
-    // Create backups directory on server if it doesn't exist
-    std::string remote_backups_dir = remotepath::backups(server_name);
-    if (!silent) std::cout << "Remote backups directory on "<< server_name <<": " << remote_backups_dir << std::endl;
-    std::string mkdir_cmd = "mkdir -p " + quote(remote_backups_dir);
-    if (!env.execute_ssh_command(mkdir_cmd)) {
-        std::cerr << "Failed to create backups directory on server" << std::endl;
-        return false;
-    }
-
-    // Create backups directory locally if it doesn't exist
-    std::string local_backups_dir = localpath::backups_path();
-    if (local_backups_dir.empty()) {
-        std::cerr << "Error: Local backups directory not found - is DropShell initialised?" << std::endl;
-        return false;
-    }
-    if (!std::filesystem::exists(local_backups_dir)) 
-        std::filesystem::create_directories(local_backups_dir);
-
-    // Get current datetime for backup filename
-    auto now = std::chrono::system_clock::now();
-    auto time = std::chrono::system_clock::to_time_t(now);
-    std::stringstream datetime;
-    datetime << std::put_time(std::localtime(&time), "%Y-%m-%d_%H-%M-%S");
-    
-    if (name_breaks_backups(server_name)) {std::cerr << "Error: Server name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
-    if (name_breaks_backups(service_name)) {std::cerr << "Error: Service name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
-    if (name_breaks_backups(service_info.template_name)) {std::cerr << "Error: Service template name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
-
-    // Construct backup filename
-    std::string backup_filename = server_name + magic_string + service_info.template_name + magic_string + service_name + magic_string + datetime.str() + ".tgz";
-    std::string remote_backup_file_path = remote_backups_dir + "/" + backup_filename;
-    std::string local_backup_file_path = (std::filesystem::path(local_backups_dir) / backup_filename).string();
-
-    // assert that the backup filename is valid - -_- appears exactly 3 times in local_backup_file_path.
-    ASSERT(3 == count_substring(magic_string, local_backup_file_path));
-
-    // Run backup script
-    if (!env.run_remote_template_command(service_name, command, {remote_backup_file_path}, silent)) {
-        std::cerr << "Backup script failed on remote server: " << remote_backup_file_path << std::endl;
-        return false;
-    }
-
-    // Copy backup file from server to local
-    std::string scp_cmd = "scp -P " + env.get_SSH_PORT() + " " + 
-                         env.get_SSH_USER() + "@" + env.get_SSH_HOST() + ":" + 
-                         quote(remote_backup_file_path) + " " + quote(local_backup_file_path) + (silent ? " > /dev/null 2>&1" : "");
-    if (!env.execute_local_command(scp_cmd)) {
-        std::cerr << "Failed to copy backup file from server" << std::endl;
-        return false;
-    }
-
-    if (!silent) {
-        std::cout << "Backup created successfully. Restore with:"<<std::endl;
-        std::cout << "   dropshell restore " << server_name << " " << service_name << " " << backup_filename << std::endl;
-    }
-    return true;
-}
-
-
-} // namespace main_commands
-
-} // namespace dropshell

src/main_commands.hpp

@@ -1,18 +0,0 @@
-#ifndef MAIN_COMMANDS_HPP
-#define MAIN_COMMANDS_HPP
-
-#include <string>
-#include <vector>
-
-namespace dropshell {
-
-    namespace main_commands {
-
-        int init(const std::vector<std::string> &args);
-        int restore(const std::vector<std::string> &args, bool silent=false);
-        int backup(const std::vector<std::string> &args, bool silent=false);
-        } // namespace main_commands
-
-} // namespace dropshell
-
-#endif

src/server_env_manager.cpp

@@ -1,44 +1,68 @@
 #include "server_env_manager.hpp"
-#include "utils/envmanager.hpp"
 #include "utils/directories.hpp"
 #include "utils/utils.hpp"
 #include "services.hpp"
-#include "contrib/base64.hpp"
 #include "templates.hpp"
+#include "utils/utils.hpp"
+#include "utils/json.hpp"
+#include "utils/execute.hpp"
 
 #include <iostream>
 #include <memory>
 #include <filesystem>
+#include <fstream>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <vector>
+#include <string>
+#include <iostream>
+#include <wordexp.h> // For potential shell-like expansion if needed
+
 
 namespace dropshell {
 
-server_env_manager::server_env_manager(const std::string& server_name) : mValid(false), mServer_name(server_name) {
+server_env_manager::server_env_manager(const std::string& server_name) : mValid(false), mServerName(server_name) {
     if (server_name.empty())
         return;
         
     // Construct the full path to server.env
-    std::string env_path = localfile::server_env(server_name);
+    std::string server_env_path = localfile::server_json(server_name);
     
     // Check if file exists
-    if (!std::filesystem::exists(env_path)) {
+    if (!std::filesystem::exists(server_env_path)) {
-        std::cerr << "Server environment file not found: " + env_path << std::endl;
+        std::cerr << "Server environment file not found: " + server_env_path << " for server " << server_name << std::endl;
         return;
     }
 
     try {
         // Use envmanager to handle the environment file
-        m_env_manager = std::unique_ptr<envmanager>(new envmanager(env_path));
+        nlohmann::json server_env_json = nlohmann::json::parse(std::ifstream(server_env_path));
-        m_env_manager->load();
+        if (server_env_json.empty()) {
+            std::cerr << "Error: Failed to parse server environment file: " + server_env_path << std::endl;
+            return;
+        }
 
-        // Get all variables
+       // get the variables from the json
-        m_env_manager->get_all_variables_substituted(variables);
+       for (const auto& var : server_env_json.items()) {
+            std::string value;
+            if (var.value().is_string())
+                value = var.value();
+            else if (var.value().is_number_integer())
+                value = std::to_string(var.value().get<int>());
+            else if (var.value().is_boolean())
+                value = var.value() ? "true" : "false";
+            else
+                value = var.value().dump();
+
+            mVariables[var.key()] = replace_with_environment_variables_like_bash(value);
+       }
 
         // Verify required variables exist
         for (const auto& var : {"SSH_HOST", "SSH_USER", "SSH_PORT", "DROPSHELL_DIR"}) {
-            if (variables.find(var) == variables.end()) {
+            if (mVariables.find(var) == mVariables.end()) {
                 // Print the variables identified in the file
                 std::cout << "Variables identified in the file:" << std::endl;
-                for (const auto& v : variables) {
+                for (const auto& v : mVariables) {
                     std::cout << "    " << v.first << std::endl;
                 }
                 throw std::runtime_error("Missing required variable: " + std::string(var));
@@ -51,84 +75,74 @@ server_env_manager::server_env_manager(const std::string& server_name) : mValid(
     }    
 }
 
+bool server_env_manager::create_server_env(const std::string &server_env_path, const std::string &SSH_HOST, const std::string &SSH_USER, const std::string &SSH_PORT, const std::string &DROPSHELL_DIR)
+{
+    nlohmann::json server_env_json;
+    server_env_json["SSH_HOST"] = SSH_HOST;
+    server_env_json["SSH_USER"] = SSH_USER;
+    server_env_json["SSH_PORT"] = SSH_PORT;
+    server_env_json["DROPSHELL_DIR"] = DROPSHELL_DIR;
+
+    try {
+        std::ofstream server_env_file(server_env_path);
+        server_env_file << server_env_json.dump(4);
+        server_env_file.close();
+        return true;
+    } catch (const std::exception& e) {
+        std::cerr << "Failed to create server environment file: " + std::string(e.what()) << std::endl;
+        return false;
+    }
+}
+
 std::string server_env_manager::get_variable(const std::string& name) const {
-    if (!m_env_manager) {
+    auto it = mVariables.find(name);
+    if (it == mVariables.end()) {
         return "";
     }
-    return m_env_manager->get_variable_substituted(name);
+    return it->second;
 }
 
-// Helper method implementations
+sCommand server_env_manager::construct_standard_template_run_cmd(const std::string &service_name, const std::string &command, std::vector<std::string> args, bool silent) const
-std::string server_env_manager::construct_ssh_cmd() const {
-    std::stringstream ssh_cmd;
-    ssh_cmd << "ssh -p " << get_SSH_PORT() << " "
-            << get_SSH_USER() << "@" << get_SSH_HOST();
-    return ssh_cmd.str();
-}
-
-std::string server_env_manager::construct_standard_command_run_cmd(const std::string &service_name, const std::string &command, std::vector<std::string> args, bool silent) const
 {
-    std::string remote_service_template_path = remotepath::service_template(mServer_name,service_name);
+    if (command.empty())
-    std::string remote_service_config_path = remotepath::service_config(mServer_name,service_name);
+        return sCommand();
 
+    std::string remote_service_template_path = remotepath::service_template(mServerName,service_name);
     std::string script_path = remote_service_template_path + "/" + command + ".sh";
 
     std::map<std::string, std::string> env_vars;
-    get_all_service_env_vars(service_name, env_vars);
+    if (!get_all_service_env_vars(mServerName, service_name, env_vars)) {
+        std::cerr << "Error: Failed to get all service env vars for " << service_name << std::endl;
+        return sCommand();
+    }
 
     std::string argstr = "";
     for (const auto& arg : args) {
         argstr += " " + quote(dequote(trim(arg)));
     }
 
-    sCommand scommand(remote_service_template_path, "bash " + quote(script_path) + argstr + (silent ? " > /dev/null 2>&1" : ""), env_vars);
+    std::vector<std::string> cmd = {script_path};
-    std::string run_cmd = scommand.construct_safecmd();
+    cmd.insert(cmd.end(), args.begin(), args.end());
-    return run_cmd;
+    sCommand scommand(remote_service_template_path, cmd, env_vars);
-}
 
-void server_env_manager::get_all_service_env_vars(const std::string &service_name, std::map<std::string, std::string> & all_env_vars) const
+    if (scommand.empty())
-{
+        std::cerr << "Error: Failed to construct command for " << service_name << " " << command << std::endl;
-    all_env_vars.clear();
 
-    // add in some handy variables.
+    return scommand;
-    all_env_vars["CONFIG_PATH"] = remotepath::service_config(mServer_name,service_name);
-    all_env_vars["SERVER"] = mServer_name;
-    all_env_vars["SERVICE"] = service_name;
-
-
-    { // load service.env from the service on this machine.
-    std::map<std::string, std::string> env_vars;
-    envmanager env_manager(localfile::service_env(mServer_name,service_name));
-    env_manager.load();
-    env_manager.get_all_variables(env_vars);
-    all_env_vars.merge(env_vars);
-    }
-
-    { // load _default.env from the template on this machine.
-    std::map<std::string, std::string> env_vars;
-    ServiceInfo service_info = get_service_info(mServer_name, service_name);
-    std::string defaultenvpath = service_info.local_template_default_env_path;
-    if (std::filesystem::exists(defaultenvpath)) {
-        envmanager env_manager(defaultenvpath);
-        env_manager.load();
-        env_manager.get_all_variables(env_vars);
-        all_env_vars.merge(env_vars);
-        }
-    else 
-        std::cerr << "Warning: _default.env not found in template: " << defaultenvpath << std::endl;
-    }
 }
 
 
 bool server_env_manager::check_remote_dir_exists(const std::string &dir_path) const
 {
-    sCommand scommand("test -d " + quote(dir_path));
+    std::vector<std::string> cmd = {"test", "-d", dir_path};
-    return execute_ssh_command(scommand);
+    sCommand scommand(cmd);
+    return execute_command(get_SSH_INFO(), scommand, cMode::Silent);
 }
 
 bool server_env_manager::check_remote_file_exists(const std::string& file_path) const {
-    sCommand scommand("test -f " + quote(file_path));
+    std::vector<std::string> cmd = {"test", "-f", file_path};
-    return execute_ssh_command(scommand);
+    sCommand scommand(cmd);
+    return execute_command(get_SSH_INFO(), scommand, cMode::Silent);
 }
 
 bool server_env_manager::check_remote_items_exist(const std::vector<std::string> &file_paths) const
@@ -141,9 +155,11 @@ bool server_env_manager::check_remote_items_exist(const std::vector<std::string>
         file_names_str += std::filesystem::path(file_path).filename().string() + " ";
     }
     // check if all items in the vector exist on the remote server, in a single command.
-    sCommand scommand("for item in " + file_paths_str + "; do test -f $item; done");
+    sCommand scommand({
+       "bash", "-c", "for item in " + file_paths_str + "; do test -f $item; done"
+    });
 
-    bool okay = execute_ssh_command(scommand);
+    bool okay = execute_command(get_SSH_INFO(), scommand, cMode::Silent);
     if (!okay) {
         std::cerr << "Error: Required items not found on remote server: " << file_names_str << std::endl;
         return false;
@@ -151,75 +167,37 @@ bool server_env_manager::check_remote_items_exist(const std::vector<std::string>
     return true;
 }
 
-bool server_env_manager::execute_ssh_command(const sCommand& command) const {
+bool server_env_manager::run_remote_template_command(const std::string &service_name, const std::string &command, std::vector<std::string> args, bool silent, std::map<std::string, std::string> extra_env_vars) const
-    std::string full_cmd = construct_ssh_cmd() + " " + quote(command.construct_safecmd());
-    return execute_local_command(full_cmd);
-}
-
-bool server_env_manager::execute_ssh_command_and_capture_output(const sCommand& command, std::string &output) const
 {
-    std::string full_cmd = construct_ssh_cmd() + " " + quote(command.construct_safecmd());
+    sCommand scommand = construct_standard_template_run_cmd(service_name, command, args, silent);
-    return execute_local_command_and_capture_output(full_cmd, output);
-}
 
-bool server_env_manager::run_remote_template_command(const std::string &service_name, const std::string &command, std::vector<std::string> args, bool silent) const
+    // add the extra env vars to the command
-{
+    for (const auto& [key, value] : extra_env_vars) 
-    std::string full_cmd = construct_standard_command_run_cmd(service_name, command, args, silent);
+            scommand.add_env_var(key, value);
-    return execute_ssh_command(full_cmd);
-}
 
-bool server_env_manager::run_remote_template_command_and_capture_output(const std::string &service_name, const std::string &command, std::vector<std::string> args, std::string &output, bool silent) const
+    if (scommand.get_command_to_run().empty())
-{
-    std::string full_cmd = construct_standard_command_run_cmd(service_name, command, args, silent);
-    return execute_ssh_command_and_capture_output(full_cmd, output);
-}
-
-bool server_env_manager::execute_local_command(const sCommand& command) {
-    return (system(command.construct_safecmd().c_str()) == 0);
-}
-
-bool server_env_manager::execute_local_command_and_capture_output(const sCommand& command, std::string &output)
-{
-    std::string full_cmd = command.construct_safecmd() + " 2>&1";
-    FILE *pipe = popen(full_cmd.c_str(), "r");
-    if (!pipe) {
         return false;
-    }
+    cMode mode = (command=="ssh" ? cMode::Defaults : cMode::Silent);
-    char buffer[128];
+    return execute_command(get_SSH_INFO(), scommand, mode);
-    while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
-        output += buffer;
-    }
-    int status = pclose(pipe);
-    return (status == 0);
 }
 
-
+bool server_env_manager::run_remote_template_command_and_capture_output(const std::string &service_name, const std::string &command, std::vector<std::string> args, std::string &output, bool silent, std::map<std::string, std::string> extra_env_vars) const
-std::string sCommand::construct_safecmd() const
 {
-    std::string to_encode;
+    sCommand scommand = construct_standard_template_run_cmd(service_name, command, args, false);
+    if (scommand.get_command_to_run().empty())
+        return false;
 
-    for (const auto& env_var : mVars) {
+    // add the extra env vars to the command
-        to_encode += env_var.first + "=" + quote(dequote(trim(env_var.second))) + " ";
+    for (const auto& [key, value] : extra_env_vars) 
+        scommand.add_env_var(key, value);
+
+    cMode mode = cMode::CaptureOutput;
+    return execute_command(get_SSH_INFO(), scommand, mode, output);
 }
-    to_encode += mCmd;
 
-    std::string encoded = base64_encode(to_encode);
-
-    std::string commandstr = "echo " + encoded + " | base64 -d | bash";
-
-    if (!mDir.empty()) 
-        commandstr = "cd " + quote(mDir) + " && " + commandstr;
-    return commandstr;
-}
 
 // base64 <<< "FOO=BAR WHEE=YAY bash ./test.sh"
 // echo YmFzaCAtYyAnRk9PPUJBUiBXSEVFPVlBWSBiYXNoIC4vdGVzdC5zaCcK | base64 -d | bash
 
-std::string makesafecmd(const std::string &command)
-{
-    std::string encoded = base64_encode(dequote(trim(command)));
-    std::string commandstr = "echo " + encoded + " | base64 -d | bash";
-    return commandstr;
-}
 
 } // namespace dropshell

src/server_env_manager.hpp

@@ -8,33 +8,14 @@
 #include <string>
 #include <map>
 #include <memory>
-#include "utils/envmanager.hpp"
+#include <vector>
+#include "utils/execute.hpp"
+
 namespace dropshell {
 
-// class to hold a command to run on the remote server.
+class server_env_manager;
-class sCommand {
-    public:
-        sCommand(std::string directory_to_run_in, std::string command_to_run, const std::map<std::string, std::string> & env_vars) : 
-            mDir(directory_to_run_in), mCmd(command_to_run), mVars(env_vars) {}
-        sCommand(std::string command_to_run) : 
-            mDir(""), mCmd(command_to_run), mVars({}) {}
 
-
+// ------------------------------------------------------------------------------------------------
-        std::string get_directory_to_run_in() const { return mDir; }
-        std::string get_command_to_run() const { return mCmd; }
-        const std::map<std::string, std::string>& get_env_vars() const { return mVars; }
-
-        void add_env_var(const std::string& key, const std::string& value) { mVars[key] = value; }
-
-        std::string construct_safecmd() const;
-
-    private:
-        std::string mDir;
-        std::string mCmd;
-        std::map<std::string, std::string> mVars;
-};
-
-std::string makesafecmd(const std::string& command);
 
 // reads path / server.env and provides a class to access the variables.
 // each env file is required to have the following variables:
@@ -46,15 +27,25 @@ std::string makesafecmd(const std::string& command);
 class server_env_manager {
     public:
         server_env_manager(const std::string& server_name);
+        
+        static bool create_server_env(
+                 const std::string& server_env_path,
+                 const std::string& SSH_HOST,
+                 const std::string& SSH_USER,
+                 const std::string& SSH_PORT,
+                 const std::string& DROPSHELL_DIR);
+
         std::string get_variable(const std::string& name) const;
 
         // trivial getters.
-        const std::map<std::string, std::string>& get_variables() const { return variables; }
+        const std::map<std::string, std::string>& get_variables() const { return mVariables; }
         std::string get_SSH_HOST() const { return get_variable("SSH_HOST"); }
         std::string get_SSH_USER() const { return get_variable("SSH_USER"); }
         std::string get_SSH_PORT() const { return get_variable("SSH_PORT"); }
         std::string get_DROPSHELL_DIR() const { return get_variable("DROPSHELL_DIR"); }
+        sSSHInfo get_SSH_INFO() const { return sSSHInfo{get_SSH_HOST(), get_SSH_USER(), get_SSH_PORT()}; }
         bool is_valid() const { return mValid; }
+        std::string get_server_name() const { return mServerName; }
 
 // helper functions
     public:
@@ -62,30 +53,22 @@ class server_env_manager {
         bool check_remote_file_exists(const std::string& file_path) const;
         bool check_remote_items_exist(const std::vector<std::string>& file_paths) const;
 
-        bool run_remote_template_command(const std::string& service_name, const std::string& command, std::vector<std::string> args, bool silent=false) const;
+        bool run_remote_template_command(const std::string& service_name, const std::string& command, 
-        bool run_remote_template_command_and_capture_output(const std::string& service_name, const std::string& command, std::vector<std::string> args, std::string & output, bool silent=false) const;
+            std::vector<std::string> args, bool silent, std::map<std::string, std::string> extra_env_vars) const;
-
+        bool run_remote_template_command_and_capture_output(const std::string& service_name, const std::string& command, 
-    public:
+            std::vector<std::string> args, std::string & output, bool silent, std::map<std::string, std::string> extra_env_vars) const;
-        bool execute_ssh_command(const sCommand& command) const;
-        bool execute_ssh_command_and_capture_output(const sCommand& command, std::string & output) const;
-
-        static bool execute_local_command(const sCommand& command);
-        static bool execute_local_command_and_capture_output(const sCommand& command, std::string & output);
 
     private:
-        std::string construct_ssh_cmd() const;
+        sCommand construct_standard_template_run_cmd(const std::string& service_name, const std::string& command, std::vector<std::string> args, bool silent) const;
-        std::string construct_standard_command_run_cmd(const std::string& service_name, const std::string& command, std::vector<std::string> args, bool silent) const;
 
     private:
-        void get_all_service_env_vars(const std::string& service_name, std::map<std::string, std::string> & all_env_vars) const;
+        std::string mServerName;
-
+        std::map<std::string, std::string> mVariables;
-    private:
-        std::string mServer_name;
-        std::map<std::string, std::string> variables;
         bool mValid;
-        std::unique_ptr<envmanager> m_env_manager;
 };
 
+
+
 } // namespace dropshell
 
 

src/servers.cpp

@@ -7,10 +7,11 @@
 #include "services.hpp"
 #include "config.hpp"
 #include "templates.hpp"
+#include "contrib/transwarp.hpp"
+
 #include <iostream>
 #include <fstream>
 #include <iomanip>
-#include <execution>
 #include <filesystem>
 
 namespace dropshell {
@@ -18,18 +19,19 @@ namespace dropshell {
 std::vector<ServerInfo> get_configured_servers() {
     std::vector<ServerInfo> servers;
 
-    std::vector<std::string> local_config_directories = gConfig().get_local_config_directories();
+    std::vector<std::string> lsdp = gConfig().get_local_server_definition_paths();
-    if (local_config_directories.empty())
+    if (lsdp.empty())
         return servers;
 
-    for (int i = 0; i < local_config_directories.size(); i++) {
+    for (auto servers_dir : lsdp) {
-        std::string servers_dir = localpath::config_servers(i);
         if (!servers_dir.empty() && std::filesystem::exists(servers_dir)) {
-
             for (const auto& entry : std::filesystem::directory_iterator(servers_dir)) {
                 if (std::filesystem::is_directory(entry)) {
                     std::string server_name = entry.path().filename().string();
 
+                    if (server_name.empty() || server_name[0]=='.' || server_name[0]=='_')
+                        continue;
+
                     server_env_manager env(server_name);
                     if (!env.is_valid()) {
                         std::cerr << "Error: Invalid server environment file: " << entry.path().string() << std::endl;
@@ -49,14 +51,38 @@ std::vector<ServerInfo> get_configured_servers() {
     return servers;
 }
 
+ServerInfo get_server_info(const std::string &server_name)
+{
+    std::vector<std::string> lsdp = gConfig().get_local_server_definition_paths();
+    if (lsdp.empty())
+        return ServerInfo();
+
+    for (auto &config_dir : lsdp) {
+        std::string server_dir = config_dir + "/" + server_name;
+        if (std::filesystem::exists(server_dir)) {
+            server_env_manager env(server_name);
+            if (!env.is_valid()) {
+                std::cerr << "Error: Invalid server environment file: " << server_dir << std::endl;
+                continue;
+            }
+            return ServerInfo({server_name, env.get_SSH_HOST(), env.get_SSH_USER(), env.get_SSH_PORT()});
+        }
+    }
+    return ServerInfo();
+}
+
+// https://github.com/bloomen/transwarp?tab=readme-ov-file#range-functions
 void list_servers() {
     auto servers = get_configured_servers();
 
     tableprint tp("All DropShell Servers");
-    tp.add_row({"Name", "Address", "Health", "Ports"});
+    tp.add_row({"Name", "User", "Address", "Health", "Ports"});
 
-    std::for_each(std::execution::par, servers.begin(), servers.end(), [&](const ServerInfo& server) {
+    std::cout << "Checking "<<servers.size() << " servers: " << std::flush;
+    int checked = 0;
 
+    transwarp::parallel exec{servers.size()};
+    auto task = transwarp::for_each(exec, servers.begin(), servers.end(), [&](const ServerInfo& server) {
         std::map<std::string, ServiceStatus> status = service_runner::get_all_services_status(server.name); 
 
         std::set<int> ports_used;
@@ -69,8 +95,13 @@ void list_servers() {
         for (const auto& port : ports_used) 
             ports_used_str += std::to_string(port) + " ";
         
-        tp.add_row({server.name, server.ssh_host, serviceticks, ports_used_str});
+        tp.add_row({server.name, server.ssh_user, server.ssh_host, serviceticks, ports_used_str});
+        ++checked;
+        // print out a tick character for each server checked.
+        std::cout << checked << " ✓  " << std::flush;
     });
+    task->wait();
+    std::cout << std::endl << std::endl;
     tp.print();
 }
 
@@ -143,19 +174,24 @@ void show_server_details(const std::string& server_name) {
     } // end of list services
 } // end of show_server_details
 
-void create_server(const std::string &server_name)
+bool create_server(const std::string &server_name)
 {
     // 1. check if server name already exists
     std::string server_existing_dir = localpath::server(server_name);
     if (!server_existing_dir.empty()) {
         std::cerr << "Error: Server name already exists: " << server_name << std::endl;
         std::cerr << "Current server path: " << server_existing_dir << std::endl;
-        return;
+        return false;
     }
 
     // 2. create a new directory in the user config directory
-    std::string config_servers_dir = localpath::config_servers();
+    auto lsdp = gConfig().get_local_server_definition_paths();
-    std::string server_dir = config_servers_dir + "/" + server_name;
+    if (lsdp.empty() || lsdp[0].empty()) {
+        std::cerr << "Error: Local server definition path not found" << std::endl;
+        std::cerr << "Run 'dropshell edit' to configure DropShell" << std::endl;
+        return false;
+    }
+    std::string server_dir = lsdp[0] + "/" + server_name;
     std::filesystem::create_directory(server_dir);
 
     // 3. create a template server.env file in the server directory
@@ -172,17 +208,13 @@ void create_server(const std::string &server_name)
     // 4. add dropshell-agent service to server
     create_service(server_name, "dropshell-agent", "dropshell-agent", true); // silently create service.
 
-    // std::string service_dir = server_dir + "/dropshell-agent";
-    // std::filesystem::create_directory(service_dir);
-    // std::string service_env_path = service_dir + "/service.env";
-    // std::filesystem::copy(get_local_system_templates_path() + "/dropshell-agent/example/service.env", service_env_path);
-
     std::cout << "Server created successfully: " << server_name << std::endl;
     std::cout << "Please complete the installation:" <<std::endl;
     std::cout << "1) edit the server configuration: dropshell edit " << server_name << std::endl;
     std::cout << "2) test ssh is working:           dropshell ssh " << server_name << std::endl;
     std::cout << "3) install dropshell-agent:       dropshell install " << server_name << " dropshell-agent" << std::endl;
     std::cout << std::endl;
+    return true;
 }
 
 void get_all_used_commands(std::set<std::string> &commands)
@@ -190,7 +222,7 @@ void get_all_used_commands(std::set<std::string> &commands)
     std::vector<ServerInfo> servers = get_configured_servers();
     for (const auto& server : servers) 
     {
-        std::vector<dropshell::ServiceInfo> services = dropshell::get_server_services_info(server.name);
+        auto services = dropshell::get_server_services_info(server.name);
         for (const auto& service : services) 
             commands.merge(dropshell::get_used_commands(server.name, service.service_name));
     }

src/servers.hpp

@@ -17,10 +17,14 @@ namespace dropshell {
     };
 
     std::vector<ServerInfo> get_configured_servers();
+
+    ServerInfo get_server_info(const std::string& server_name);
+
+
     void list_servers();
     void show_server_details(const std::string& server_name);
 
-    void create_server(const std::string& server_name);
+    bool create_server(const std::string& server_name);
    
     void get_all_used_commands(std::set<std::string> &commands);
 

src/service_runner.cpp

@@ -1,10 +1,3 @@
-#include "config.hpp"
-#include "service_runner.hpp"
-#include "server_env_manager.hpp"
-#include "templates.hpp"
-#include "services.hpp"
-#include "utils/directories.hpp"
-#include "utils/utils.hpp"
 #include <iostream>
 #include <fstream>
 #include <sstream>
@@ -13,11 +6,23 @@
 #include <iomanip>
 #include <filesystem>
 #include <unistd.h>
+#include <libassert/assert.hpp>
+
+#include "config.hpp"
+#include "service_runner.hpp"
+#include "server_env_manager.hpp"
+#include "templates.hpp"
+#include "services.hpp"
+#include "utils/directories.hpp"
+#include "utils/utils.hpp"
+
 
 namespace fs = std::filesystem;
 
 namespace dropshell {
 
+static const std::string magic_string = "-_-";
+
 service_runner::service_runner(const std::string& server_name, const std::string& service_name) : 
     mServerEnv(server_name), mServer(server_name), mService(service_name), mValid(false)
 {
@@ -29,33 +34,34 @@ service_runner::service_runner(const std::string& server_name, const std::string
         return;
 
     mServiceInfo = get_service_info(server_name, service_name);
+    if (mServiceInfo.service_name.empty())
+        return;
+
     mService = mServiceInfo.service_name;
     
     mValid = !mServiceInfo.local_template_path.empty();
 }
 
-bool service_runner::install() {
+bool service_runner::install(bool silent) {
     maketitle("Installing " + mService + " (" + mServiceInfo.template_name + ") on " + mServer);
 
     if (!mServerEnv.is_valid()) return false; // should never hit this.
     
     // Check if template exists
-    template_info tinfo;
+    template_info tinfo = gTemplateManager().get_template_info(mServiceInfo.template_name);
-    if (!get_template_info(mServiceInfo.template_name, tinfo))
+    if (!tinfo.is_set())
         return false;
         
     // Create service directory
     std::string remote_service_path = remotepath::service(mServer, mService);
-    std::string mkdir_cmd = "mkdir -p " + quote(remote_service_path);
+    if (!execute_command(mServerEnv.get_SSH_INFO(), sCommand({"mkdir", "-p", quote(remote_service_path)}), cMode::Silent))
-    if (!mServerEnv.execute_ssh_command(mkdir_cmd))
     {
         std::cerr << "Failed to create service directory " << remote_service_path << std::endl;
         return false;
     }
     
     // Check if rsync is installed on remote host
-    std::string check_rsync_cmd = "which rsync > /dev/null 2>&1";
+    if (!execute_command(mServerEnv.get_SSH_INFO(), sCommand({"which", "rsync", ">", "/dev/null", "2>&1"}), cMode::Silent))
-    if (!mServerEnv.execute_ssh_command(check_rsync_cmd))
     {
         std::cerr << "rsync is not installed on the remote host" << std::endl;
         return false;
@@ -63,21 +69,19 @@ bool service_runner::install() {
 
     // Copy template files
     {
-        std::cout << "Copying: [LOCAL] " << tinfo.local_template_path << std::endl << std::string(8,' ')<<"[REMOTE] " << remotepath::service_template(mServer, mService) << "/" << std::endl;
+        std::cout << "Copying: [LOCAL] " << tinfo.local_template_path() << std::endl << std::string(8,' ')<<"[REMOTE] " << remotepath::service_template(mServer, mService) << "/" << std::endl;
-        std::string rsync_cmd = "rsync --delete -zrpc -e 'ssh -p " + mServerEnv.get_SSH_PORT() + "' " + 
+        std::vector<std::string> rsync_cmd = {"rsync", "--delete", "-zrpc", "-e", quote("ssh -p " + mServerEnv.get_SSH_PORT()), 
-                            quote(tinfo.local_template_path + "/") + " "+ 
+                            quote(tinfo.local_template_path().string()+"/"), 
-                            mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + 
+                            mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + quote(remotepath::service_template(mServer, mService)+"/")
-                            quote(remotepath::service_template(mServer, mService)+"/");
+                            };
-        //std::cout << std::endl << rsync_cmd << std::endl << std::endl;
+        if (!execute_command(rsync_cmd, silent ? cMode::Silent : cMode::Defaults))
-        if (!mServerEnv.execute_local_command(rsync_cmd))
         {
             std::cerr << "Failed to copy template files using rsync" << std::endl;
-            std::cerr << "Is rsync installed on the remote host?" << std::endl;
             return false;
         }
     }
     
-    // Copy service files (including service.env)
+    // Copy service files
     {
         std::string local_service_path = localpath::service(mServer,mService);
         if (local_service_path.empty() || !fs::exists(local_service_path)) {
@@ -85,11 +89,11 @@ bool service_runner::install() {
             return false;
         }
         std::cout << "Copying: [LOCAL] " << local_service_path << std::endl <<std::string(8,' ')<<"[REMOTE] " << remotepath::service_config(mServer,mService) << std::endl; 
-        std::string rsync_cmd = "rsync --delete -zrpc -e 'ssh -p " + mServerEnv.get_SSH_PORT() + "' " + 
+        std::vector<std::string> rsync_cmd = {"rsync", "--delete", "-zrpc", "-e", quote("ssh -p " + mServerEnv.get_SSH_PORT()), 
-                            quote(local_service_path + "/") + " "+
+                            quote(local_service_path + "/"), 
-                            mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + 
+                            mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + quote(remotepath::service_config(mServer,mService) + "/")
-                            quote(remotepath::service_config(mServer,mService) + "/");
+                            };
-        if (!mServerEnv.execute_local_command(rsync_cmd))
+        if (!execute_command(rsync_cmd, silent ? cMode::Silent : cMode::Defaults))
         {
             std::cerr << "Failed to copy service files using rsync" << std::endl;
             return false;
@@ -98,7 +102,7 @@ bool service_runner::install() {
 
     // Run install script
     {
-        mServerEnv.run_remote_template_command(mService, "install", {});
+        mServerEnv.run_remote_template_command(mService, "install", {}, silent, {});
     }
 
     // print health tick
@@ -106,7 +110,7 @@ bool service_runner::install() {
     return true;
 }
 
-bool service_runner::uninstall() {
+bool service_runner::uninstall(bool silent) {
     maketitle("Uninstalling " + mService + " (" + mServiceInfo.template_name + ") on " + mServer);
     
     if (!mServerEnv.is_valid()) return false; // should never hit this.
@@ -118,11 +122,11 @@ bool service_runner::uninstall() {
     }
     
     // 3. Run uninstall script if it exists
-    std::string uninstall_script = remotepath::service_template(mServer, mService) + "/_uninstall.sh";
+    std::string uninstall_script = remotepath::service_template(mServer, mService) + "/uninstall.sh";
     bool script_exists = mServerEnv.check_remote_file_exists(uninstall_script);
     
     if (script_exists) {
-        if (!mServerEnv.run_remote_template_command(mService, "uninstall", {})) {
+        if (!mServerEnv.run_remote_template_command(mService, "uninstall", {}, silent, {})) {
             std::cerr << "Warning: Uninstall script failed, but continuing with directory removal" << std::endl;
         }
 
@@ -132,8 +136,8 @@ bool service_runner::uninstall() {
     }
     
     // 4. Remove the service directory from the server
-    std::string rm_cmd = "'rm -rf " + quote(remotepath::service(mServer, mService)) + "'";
+    std::vector<std::string> rm_cmd = {"rm", "-rf", remotepath::service(mServer, mService)};
-    if (!mServerEnv.execute_ssh_command(rm_cmd)) {
+    if (!execute_command(mServerEnv.get_SSH_INFO(), sCommand(rm_cmd), cMode::Silent)) {
         std::cerr << "Failed to remove service directory" << std::endl;
         return false;
     }
@@ -142,17 +146,64 @@ bool service_runner::uninstall() {
     return true;
 }
 
+bool service_runner::nuke(bool silent)
+{
+    maketitle("Nuking " + mService + " (" + mServiceInfo.template_name + ") on " + mServer);
+
+    if (!mServerEnv.is_valid()) return false; // should never hit this.
+
+    std::string remote_service_path = remotepath::service(mServer, mService);
+    bool okay = mServerEnv.run_remote_template_command("dropshell-agent", "_nuke_other", {mService, remote_service_path}, silent, {});
+    if (!okay)
+    {
+        std::cerr << "Warning: Nuke script failed" << std::endl;
+        return false;
+    }
+
+    std::cout << "Service " << mService << " successfully nuked from " << mServer << std::endl;
+
+    if (!silent) {
+        std::cout << "There's nothing left on the remote server." << std::endl;
+        std::cout << "You can remove the local files with:" << std::endl;
+        std::cout << "   rm -rf " << localpath::service(mServer,mService) << std::endl;
+    }
+    return true;
+}
+
+bool service_runner::fullnuke()
+{
+    if (!nuke(true))
+    {
+        std::cerr << "Warning: Nuke script failed, aborting fullnuke!" << std::endl;
+        return false;
+    }
+
+    std::string local_service_path = mServiceInfo.local_service_path;
+    if (local_service_path.empty() || !fs::exists(local_service_path)) {
+        std::cerr << "Error: Service directory not found: " << local_service_path << std::endl;
+        return false;
+    }  
+    
+    std::vector<std::string> rm_cmd = {"rm", "-rf", local_service_path};
+    if (!execute_command(sCommand(rm_cmd), cMode::Silent)) {
+        std::cerr << "Failed to remove service directory" << std::endl;
+        return false;
+    }
+    
+    return true;
+}
+
 
 // ------------------------------------------------------------------------------------------------
 // Run a command on the service.
 // ------------------------------------------------------------------------------------------------
-bool service_runner::run_command(const std::string& command) {
+bool service_runner::run_command(const std::string& command, std::vector<std::string> additional_args, std::map<std::string, std::string> env_vars) {
     if (!mServerEnv.is_valid()) {
         std::cerr << "Error: Server service not initialized" << std::endl;
         return false;
     }
-    template_info tinfo;
+    template_info tinfo = gTemplateManager().get_template_info(mServiceInfo.template_name);
-    if (!get_template_info(mServiceInfo.template_name, tinfo)) {
+    if (!tinfo.is_set()) {
         std::cerr << "Error: Template '" << mServiceInfo.template_name << "' not found" << std::endl;
         return false;
     }
@@ -163,7 +214,16 @@ bool service_runner::run_command(const std::string& command) {
         return true;
     }
 
-    if (!template_command_exists(mServiceInfo.template_name, command)) {
+    if (command == "fullnuke") 
+        return fullnuke();
+
+    if (command == "nuke") 
+    {
+        std::cout << "Nuking " << mService << " (" << mServiceInfo.template_name << ") on " << mServer << std::endl;
+        return nuke();
+    }
+
+    if (!gTemplateManager().template_command_exists(mServiceInfo.template_name, command)) {
         std::cout << "No command script for " << mServiceInfo.template_name << " : " << command << std::endl;
         return true; // nothing to run.
     }
@@ -176,29 +236,44 @@ bool service_runner::run_command(const std::string& command) {
     
     // Check if service directory exists
     if (!mServerEnv.check_remote_dir_exists(remotepath::service(mServer, mService))) {
+        std::cerr << "Error: Service is not installed: " << mService << std::endl;
         return false;
     }
     
     // Check if command script exists
     if (!mServerEnv.check_remote_file_exists(script_path)) {
+        std::cerr << "Error: Remote command script not found: " << script_path << std::endl;
         return false;
     }
 
     // Check if env file exists
     if (!mServerEnv.check_remote_file_exists(remotefile::service_env(mServer, mService))) {
+        std::cerr << "Error: Service config file not found: " << remotefile::service_env(mServer, mService) << std::endl;
         return false;
     }
 
     if (command == "uninstall") 
         return uninstall();
+
     if (command == "ssh") {
         interactive_ssh_service();
         return true;
     }
+    if (command == "restore") {
+        if (additional_args.size() < 1) {
+            std::cerr << "Error: restore requires a backup file:" << std::endl;
+            std::cerr << "dropshell restore <server> <service> <backup-file>" << std::endl;
+            return false;
+        }
+        return restore(additional_args[0], false);
+    }
+    if (command == "backup") {
+        return backup(false);
+    }
 
     // Run the generic command
     std::vector<std::string> args; // not passed through yet.
-    return mServerEnv.run_remote_template_command(mService, command, args);
+    return mServerEnv.run_remote_template_command(mService, command, args, false, env_vars);
 }
 
 
@@ -209,7 +284,7 @@ std::map<std::string, ServiceStatus> service_runner::get_all_services_status(std
     std::string command = "_allservicesstatus";
     std::string service_name = "dropshell-agent";
 
-    if (!template_command_exists(service_name, command))
+    if (!gTemplateManager().template_command_exists(service_name, command))
         {
             std::cerr << "Error: " << service_name << " does not contain the " << command << " script" << std::endl;
             return status;
@@ -222,7 +297,7 @@ std::map<std::string, ServiceStatus> service_runner::get_all_services_status(std
     }
 
     std::string output;
-    if (!env.run_remote_template_command_and_capture_output(service_name, command, {}, output))
+    if (!env.run_remote_template_command_and_capture_output(service_name, command, {}, output, true, {}))
         return status;
 
     std::stringstream ss(output);
@@ -276,7 +351,7 @@ HealthStatus service_runner::is_healthy()
     }
     
     // Run status script, does not display output.
-    if (!mServerEnv.run_remote_template_command(mService, "status", {}, true))
+    if (!mServerEnv.run_remote_template_command(mService, "status", {}, true, {}))
         return HealthStatus::UNHEALTHY;
     return HealthStatus::HEALTHY;
 }
@@ -313,33 +388,6 @@ std::string service_runner::HealthStatus2String(HealthStatus status)
         return ":error:";
 }
 
-bool service_runner::ensure_service_dropshell_files_up_to_date()
-{
-    if (!mServerEnv.is_valid()) {
-        std::cerr << "Error: Server service not initialized" << std::endl;
-        return false;
-    }
-    
-    // check if the service template and config are up to date on the remote server.
-    service_versions versions(mServer, mService);
-    if (versions.remote_up_to_date())
-        return true;
-    
-    if (!versions.remote_template_is_up_to_date()) {
-        std::cerr << "Error: Service template is not up to date on the remote server" << std::endl;
-        return false;
-    }
-
-    if (!versions.remote_config_is_up_to_date()) {
-        std::cerr << "Error: Service config is not up to date on the remote server" << std::endl;
-        return false;
-    }
-
-// TODO - actually update things!
-
-    versions.update_stored_remote_versions();
-    return versions.remote_up_to_date();
-}
 
 std::string service_runner::healthmark()
 {
@@ -347,37 +395,23 @@ std::string service_runner::healthmark()
     return HealthStatus2String(status);
 }
 
-void interactive_ssh(const std::string & server_name, const std::string & command) {
+bool service_runner::interactive_ssh(const std::string & server_name, const std::string & command) {
     std::string serverpath = localpath::server(server_name);
     if (serverpath.empty()) {
         std::cerr << "Error: Server not found: " << server_name << std::endl;
-        return;
+        return false;
     }
     
+    sCommand scommand({"bash"});
     server_env_manager env(server_name);
     if (!env.is_valid()) {
         std::cerr << "Error: Invalid server environment file: " << server_name << std::endl;
-        return;
+        return false;
+    }
+    return execute_command(env.get_SSH_INFO(), scommand, cMode::Defaults);
 }
 
-    std::string ssh_address = env.get_SSH_HOST();
+void service_runner::edit_server(const std::string &server_name)
-    std::string ssh_user = env.get_SSH_USER();
-    std::string ssh_port = env.get_SSH_PORT();
-
-    std::string login = ssh_user + "@" + ssh_address;
-
-    // Execute ssh with server_name and command
-    if (command.empty())
-        execlp("ssh", "ssh", "-tt", login.c_str(), "-p", ssh_port.c_str(), nullptr);
-    else
-        execlp("ssh", "ssh", "-tt", login.c_str(), "-p", ssh_port.c_str(), command.c_str(), nullptr);
-    
-    // If exec returns, it means there was an error
-    perror("ssh execution failed");
-    exit(EXIT_FAILURE);
-}
-
-void edit_server(const std::string &server_name)
 {
     std::string serverpath = localpath::server(server_name);
     if (serverpath.empty()) {
@@ -391,46 +425,50 @@ void edit_server(const std::string &server_name)
               << "Once moved, reinstall all services with:   dropshell install " << server_name;
 
     std::string config_file = serverpath + "/server.env";
-    edit_file(config_file, aftertext.str());
+    if (!edit_file(config_file)) {
+        std::cerr << "Error: Failed to edit server.env" << std::endl;
+        std::cerr << "You can manually edit this file at: " << config_file << std::endl;
+        std::cerr << "After editing, " << aftertext.str() << std::endl;
+    }
+    else
+        std::cout << aftertext.str() << std::endl;
 }
 
-void edit_file(const std::string &file_path, const std::string & aftertext)
+bool service_runner::edit_file(const std::string &file_path)
 {
-    std::string cmd = "nano -w "+file_path+" ; echo \""+aftertext+"\"";
+    // make sure parent directory exists.
-    execlp("bash","bash","-c",cmd.c_str(), nullptr);
+    std::string parent_dir = get_parent(file_path);
+    std::filesystem::create_directories(parent_dir);
 
-    // If exec returns, it means there was an error
+    std::vector<std::string> editor_cmd;
-    perror("ssh execution failed");
+    const char* editor_env = std::getenv("EDITOR");
-    exit(EXIT_FAILURE);
+
+    if (editor_env && std::strlen(editor_env) > 0) {
+        editor_cmd = {std::string(editor_env), file_path};
+    } else if (isatty(STDIN_FILENO)) {
+        // Check if stdin is connected to a terminal if EDITOR is not set
+        editor_cmd = {"nano", "-w", file_path};
+    } else {
+        std::cerr << "Error: Standard input is not a terminal and EDITOR environment variable is not set." << std::endl;
+        std::cerr << "Try setting the EDITOR environment variable (e.g., export EDITOR=nano) or run in an interactive terminal." << std::endl;
+        std::cerr << "You can manually edit the file at: " << file_path << std::endl;
+        return false;
     }
 
-bool service_runner::restore(std::string backup_file)
+    std::cout << "Editing file: " << file_path << std::endl;
-{
+    return execute_command(editor_cmd, cMode::Defaults);
-    std::string command = "restore";
-    std::string script_path = remotepath::service_template(mServer, mService) + "/" + command + ".sh";
-    if (!template_command_exists(mServiceInfo.template_name, command)) {
-        std::cout << "No restore script for " << mServiceInfo.template_name << std::endl;
-        return true; // nothing to restore.
 }
 
-    /// TOODOOOOOO!!!!!!
+bool service_runner::interactive_ssh_service()
-    std::cout << "Restore not implemented yet" << std::endl;
-    return true;
-
-    //    std::string run_cmd = construct_standard_command_run_cmd("restore");
-    //    return execute_ssh_command(run_cmd, "Restore script failed");
-}
-
-void service_runner::interactive_ssh_service()
 {
     std::set<std::string> used_commands = get_used_commands(mServer, mService);
     if (used_commands.find("ssh") == used_commands.end()) {
         std::cerr << "Error: "<< mService <<" does not support ssh" << std::endl;
-        return;
+        return false;
     }
 
     std::vector<std::string> args; // not passed through yet.
-    mServerEnv.run_remote_template_command(mService, "ssh", args);
+    return mServerEnv.run_remote_template_command(mService, "ssh", args, false, {});
 }
 
 void service_runner::edit_service_config()
@@ -441,9 +479,291 @@ void service_runner::edit_service_config()
         return;
     }
 
-    std::string aftertext = "To apply your changes, run:\n   dropshell install " + mServer + " " + mService;
+    if (edit_file(config_file) && std::filesystem::exists(config_file))    
+        std::cout << "To apply your changes, run:\n   dropshell install " + mServer + " " + mService << std::endl;
+}
 
-    edit_file(config_file, aftertext);
+
+
+bool service_runner::restore(std::string backup_file, bool silent)
+{
+    if (backup_file.empty()) {
+        std::cerr << "Error: not enough arguments.  dropshell restore <server> <service> <backup-file>" << std::endl;
+        return false;
+    }
+
+    std::string local_backups_dir = gConfig().get_local_backup_path();
+
+    if (backup_file == "latest") {
+        // get the latest backup file from the server
+        backup_file = get_latest_backup_file(mServer, mService);
+    }
+
+    std::string local_backup_file_path = (std::filesystem::path(local_backups_dir) / backup_file).string();
+
+    if (! std::filesystem::exists(local_backup_file_path)) {
+        std::cerr << "Error: Backup file not found at " << local_backup_file_path << std::endl;
+        return false;
+    }
+
+    // split the backup filename into parts based on the magic string
+    std::vector<std::string> parts = dropshell::split(backup_file, magic_string);
+    if (parts.size() != 4) {
+        std::cerr << "Error: Backup file format is incompatible, - in one of the names?" << std::endl;
+        return false;
+    }
+
+    std::string backup_server_name = parts[0];
+    std::string backup_template_name = parts[1];
+    std::string backup_service_name = parts[2];
+    std::string backup_datetime = parts[3];
+
+    if (backup_template_name != mServiceInfo.template_name) {
+        std::cerr << "Error: Backup template does not match service template. Can't restore." << std::endl;
+        return false;
+    }
+
+    std::string nicedate = std::string(backup_datetime).substr(0, 10);
+
+    std::cout << "Restoring " << nicedate << " backup of " << backup_template_name << " taken from "<<backup_server_name<<", onto "<<mServer<<"/"<<mService<<std::endl;
+    std::cout << std::endl;
+    std::cout << "*** ALL DATA FOR "<<mServer<<"/"<<mService<<" WILL BE OVERWRITTEN! ***"<<std::endl;
+
+    // run the restore script
+    std::cout << "OK, here goes..." << std::endl;
+
+    { // backup existing service
+        maketitle("1) Backing up old service... ");
+        if (!backup(true)) // silent=true
+        {
+            std::cerr << std::endl;
+            std::cerr << "Error: Backup failed, restore aborted." << std::endl;
+            std::cerr << "You can try using   dropshell install "<<mServer<<" "<<mService<<"   to install the service afresh." << std::endl;
+            std::cerr << "Otherwise, stop the service, create and initialise a new one, then restore to that." << std::endl;
+            return false;
+        }
+        std::cout << "Backup complete." << std::endl;
+    }
+
+    { // uninstall service, then nuke it.
+        maketitle("2) Uninstalling old service...");
+        if (!uninstall(true))
+            return false;
+
+        maketitle("3) Nuking old service...");
+        if (!nuke(true))
+            return false;
+    }
+
+
+    { // restore service from backup
+        maketitle("4) Restoring service data from backup...");
+        std::string remote_backups_dir = remotepath::backups(mServer);
+        std::string remote_backup_file_path = remote_backups_dir + "/" + backup_file;
+
+        // Copy backup file from local to server
+        std::vector<std::string> scp_cmd = {"scp", "-P", mServerEnv.get_SSH_PORT(), quote(local_backup_file_path), mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + quote(remote_backup_file_path) + (silent ? " > /dev/null 2>&1" : "")};
+        if (!execute_command(sCommand(scp_cmd), silent ? cMode::Silent : cMode::Defaults)) {
+            std::cerr << "Failed to copy backup file from server" << std::endl;
+            return false;
+        }
+
+        cRemoteTempFolder remote_temp_folder(mServerEnv);
+        mServerEnv.run_remote_template_command(mService, "restore", {}, silent, {{"BACKUP_FILE", remote_backup_file_path}, {"TEMP_DIR", remote_temp_folder.path()}});
+    } // dtor of remote_temp_folder will clean up the temp folder on the server
+
+
+    { // installing fresh service
+        maketitle("5) Non-destructive install of fresh service...");
+        if (!install(true))
+            return false;
+    }
+
+    bool healthy = false;
+    {// healthcheck the service
+        maketitle("6) Healthchecking service...");
+        std::string green_tick = "\033[32m✓\033[0m";
+        std::string red_cross = "\033[31m✗\033[0m";    
+        healthy= (mServerEnv.run_remote_template_command(mService, "status", {}, silent, {}));
+        if (!silent)
+            std::cout << (healthy ? green_tick : red_cross) << " Service is " << (healthy ? "healthy" : "NOT healthy") << std::endl;
+    }
+
+    return healthy;
+}
+
+
+bool name_breaks_backups(std::string name)
+{
+    // if name contains -_-, return true
+    return name.find("-_-") != std::string::npos;
+}
+
+// backup the service over ssh, using the credentials from server.env (via server_env.hpp)
+// 1. run backup.sh on the server
+// 2. create a backup file with format server-service-datetime.tgz
+// 3. store it in the server's DROPSHELL_DIR/backups folder
+// 4. copy it to the local user_dir/backups folder
+
+// ------------------------------------------------------------------------------------------------
+// Backup the service.
+// ------------------------------------------------------------------------------------------------
+bool service_runner::backup(bool silent) {
+    auto service_info = get_service_info(mServer, mService);
+    if (service_info.local_service_path.empty()) {
+        std::cerr << "Error: Service not found" << std::endl;
+        return 1;
+    }
+
+    const std::string command = "backup";
+
+    if (!gTemplateManager().template_command_exists(service_info.template_name, command)) {
+        std::cout << "No backup script for " << service_info.template_name << std::endl;
+        return true; // nothing to back up.
+    }
+    
+    // Check if basic installed stuff is in place.
+    std::string remote_service_template_path = remotepath::service_template(mServer, mService);
+    std::string remote_command_script_file = remote_service_template_path + "/" + command + ".sh";
+    std::string remote_service_config_path = remotepath::service_config(mServer, mService);
+    if (!mServerEnv.check_remote_items_exist({
+        remotepath::service(mServer, mService),
+        remote_command_script_file,
+        remotefile::service_env(mServer, mService)})
+    )
+    {
+        std::cerr << "Error: Required service directories not found on remote server" << std::endl;
+        std::cerr << "Is the service installed?" << std::endl;
+        return false;
+    }
+
+    // Create backups directory on server if it doesn't exist
+    std::string remote_backups_dir = remotepath::backups(mServer);
+    if (!silent) std::cout << "Remote backups directory on "<< mServer <<": " << remote_backups_dir << std::endl;
+    if (!execute_command(mServerEnv.get_SSH_INFO(), sCommand({"mkdir","-p",remote_backups_dir}), cMode::Silent)) {
+        std::cerr << "Failed to create backups directory on server" << std::endl;
+        return false;
+    }
+
+    // Create backups directory locally if it doesn't exist
+    std::string local_backups_dir = gConfig().get_local_backup_path();
+    if (local_backups_dir.empty()) {
+        std::cerr << "Error: Local backups directory not found" << std::endl;
+        std::cerr << "Run 'dropshell edit' to configure DropShell" << std::endl;
+        return false;
+    }
+    if (!std::filesystem::exists(local_backups_dir)) 
+        std::filesystem::create_directories(local_backups_dir);
+
+    // Get current datetime for backup filename
+    auto now = std::chrono::system_clock::now();
+    auto time = std::chrono::system_clock::to_time_t(now);
+    std::stringstream datetime;
+    datetime << std::put_time(std::localtime(&time), "%Y-%m-%d_%H-%M-%S");
+    
+    if (name_breaks_backups(mServer)) {std::cerr << "Error: Server name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
+    if (name_breaks_backups(mService)) {std::cerr << "Error: Service name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
+    if (name_breaks_backups(service_info.template_name)) {std::cerr << "Error: Service template name contains invalid character sequence ( -_- ) that would break backup naming scheme" << std::endl; return 1;}
+
+    // Construct backup filename
+    std::string backup_filename = mServer + magic_string + service_info.template_name + magic_string + mService + magic_string + datetime.str() + ".tgz";
+    std::string remote_backup_file_path = remote_backups_dir + "/" + backup_filename;
+    std::string local_backup_file_path = (std::filesystem::path(local_backups_dir) / backup_filename).string();
+
+    // assert that the backup filename is valid - -_- appears exactly 3 times in local_backup_file_path.
+    ASSERT(3 == count_substring(magic_string, local_backup_file_path));
+
+    {   // Run backup script
+        cRemoteTempFolder remote_temp_folder(mServerEnv);
+        if (!mServerEnv.run_remote_template_command(mService, command, {}, silent, {{"BACKUP_FILE", remote_backup_file_path}, {"TEMP_DIR", remote_temp_folder.path()}})) {
+            std::cerr << "Backup script failed on remote server: " << remote_backup_file_path << std::endl;
+            return false;
+        }
+
+        // Copy backup file from server to local
+        std::vector<std::string> scp_cmd = {"scp", "-P", mServerEnv.get_SSH_PORT(), 
+                            mServerEnv.get_SSH_USER() + "@" + mServerEnv.get_SSH_HOST() + ":" + 
+                            remote_backup_file_path, 
+                            local_backup_file_path};
+        if (!execute_command(sCommand(scp_cmd), silent ? cMode::Silent : cMode::Defaults)) {
+            std::cerr << "Failed to copy backup file from server" << std::endl;
+            return false;
+        }
+    } // dtor of remote_temp_folder will clean up the temp folder on the server
+
+    if (!silent) {
+        std::cout << "Backup created successfully. Restore with:"<<std::endl;
+        std::cout << "   dropshell restore " << mServer << " " << mService << " " << backup_filename << std::endl;
+    }
+    return true;
+}
+
+cRemoteTempFolder::cRemoteTempFolder(const server_env_manager &server_env) : mServerEnv(server_env)
+{
+    std::string p = remotepath::temp_files(server_env.get_server_name()) + "/" + random_alphanumeric_string(10);
+    std::vector<std::string> mkdir_cmd = {"mkdir", "-p", p};
+    if (!execute_command(server_env.get_SSH_INFO(), sCommand(mkdir_cmd), cMode::Silent)) 
+        std::cerr << "Failed to create temp directory on server" << std::endl;
+    else 
+        mPath = p;
+}
+
+cRemoteTempFolder::~cRemoteTempFolder()
+{
+    std::vector<std::string> rm_cmd = {"rm", "-rf", mPath};
+    execute_command(mServerEnv.get_SSH_INFO(), sCommand(rm_cmd), cMode::Silent);
+}
+
+std::string cRemoteTempFolder::path() const
+{
+    return mPath;
+}
+
+// Helper function to get the latest backup file for a given server and service
+std::string service_runner::get_latest_backup_file(const std::string& server, const std::string& service) {
+    std::string local_backups_dir = gConfig().get_local_backup_path();
+    if (local_backups_dir.empty() || !std::filesystem::exists(local_backups_dir)) {
+        std::cerr << "Error: Local backups directory not found: " << local_backups_dir << std::endl;
+        return "";
+    }
+
+    // Get the template name for this service
+    LocalServiceInfo info = get_service_info(server, service);
+    if (info.template_name.empty()) {
+        std::cerr << "Error: Could not determine template name for service: " << service << std::endl;
+        return "";
+    }
+
+    // Build the expected prefix for backup files
+    std::string prefix = server + magic_string + info.template_name + magic_string + service + magic_string;
+    std::string latest_file;
+    std::string latest_datetime;
+
+    std::cout << "Looking for backup files in " << local_backups_dir << std::endl;
+
+    for (const auto& entry : std::filesystem::directory_iterator(local_backups_dir)) {
+        if (!entry.is_regular_file()) continue;
+        std::string filename = entry.path().filename().string();
+        if (filename.rfind(prefix, 0) == 0) { // starts with prefix
+            // Extract the datetime part
+            size_t dt_start = prefix.size();
+            size_t dt_end = filename.find(".tgz", dt_start);
+            if (dt_end == std::string::npos) continue;
+            std::string datetime = filename.substr(dt_start, dt_end - dt_start);
+            std::cout << "Found backup file: " << filename << " with datetime: " << datetime << std::endl;
+            if (datetime > latest_datetime) {
+                latest_datetime = datetime;
+                latest_file = filename;
+            }
+        }
+    }
+
+    if (latest_file.empty()) {
+        std::cerr << "Error: No backup files found for " << server << ", " << service << std::endl;
+    }
+
+    std::cout << "Latest backup file: " << latest_file << std::endl;
+    return latest_file;
 }
 
 } // namespace dropshell

src/service_runner.hpp

@@ -12,7 +12,6 @@
 #include "server_env_manager.hpp"
 #include "services.hpp"
 #include "utils/utils.hpp"
-#include "utils/assert.hpp"
 #include "utils/hash.hpp"
 
 namespace dropshell {
@@ -30,6 +29,7 @@ typedef struct ServiceStatus {
     std::vector<int> ports;
 } ServiceStatus;
 
+
 class service_runner {
     public:
         service_runner(const std::string& server_name, const std::string& service_name);
@@ -44,7 +44,7 @@ class service_runner {
         // checking that the command exists in the service directory.
         // checking that the command is a valid .sh file.
         // checking that the {service_name}.env file exists in the service directory.
-        bool run_command(const std::string& command);
+        bool run_command(const std::string& command, std::vector<std::string> additional_args={}, std::map<std::string, std::string> env_vars={});
 
         // check health of service. Silent.
         // 1. run status.sh on the server
@@ -55,13 +55,6 @@ class service_runner {
         std::string healthtick();
         std::string healthmark();
 
-        // get the status of all services on the server
-        static std::map<std::string, ServiceStatus> get_all_services_status(std::string server_name);
-        static std::string HealthStatus2String(HealthStatus status);
-
-        // ensure the service related dropshell files (template and config) are up to date on the remote server.
-        bool ensure_service_dropshell_files_up_to_date();
-
     private:    
         // install the service over ssh, using the credentials from server.env (via server_env.hpp), by:
         // 1. check if the server_name exists, and the service_name refers to a valid template
@@ -70,31 +63,44 @@ class service_runner {
         // 3. copy the template files into {DROPSHELL_DIR}/{service_name}/template (from the templates directory for the specified server, using templates.hpp to identify the path)
         // 4. copying the local service directory into {DROPSHELL_DIR}/{service_name}/config (from the server directory for the specified server)
         // 5. running the install.sh script on the server, passing the {service_name}.env file as an argument
-        bool install(); 
+        bool install(bool silent=false); 
         
         // uninstall the service over ssh, using the credentials from server.env (via server_env.hpp)
         // 1. check if the server_name exists, and the service_name refers to a valid template
         // 2. check if service_name is valid for the server_name
         // 3. run the uninstall.sh script on the server, passing the {service_name}.env file as an argument
         // 4. remove the service directory from the server
-        bool uninstall();
+        bool uninstall(bool silent=false);
 
-        // restore the service over ssh, using the credentials from server.env (via server_env.hpp)
+        // backup and restore
-        // 1. copy the backup file to the server's DROPSHELL_DIR/backups folder
+        bool backup(bool silent=false);
-        // 2. run the restore.sh script on the server, passing the {service_name}.env file as an argument
+        bool restore(std::string backup_file, bool silent=false);
-        bool restore(std::string backup_file);
+
+        // nuke the service
+        bool nuke(bool silent=false); // nukes all data for this service on the remote server
+        bool fullnuke(); // nuke all data for this service on the remote server, and then nukes all the local service definitionfiles
 
         // launch an interactive ssh session on a server or service
         // replaces the current dropshell process with the ssh process
-        void interactive_ssh_service();
+        bool interactive_ssh_service();
 
         // edit the service configuration file
         void edit_service_config();
 
+
+    public:
+        // utility functions
+        static std::string get_latest_backup_file(const std::string& server, const std::string& service);
+        static bool interactive_ssh(const std::string & server_name, const std::string & command);
+        static void edit_server(const std::string & server_name);
+        static bool edit_file(const std::string & file_path);
+        static std::map<std::string, ServiceStatus> get_all_services_status(std::string server_name);
+        static std::string HealthStatus2String(HealthStatus status);
+
     private:
         std::string mServer;
         server_env_manager mServerEnv;
-        ServiceInfo mServiceInfo;
+        LocalServiceInfo mServiceInfo;
         std::string mService;
         bool mValid;
 
@@ -102,39 +108,16 @@ class service_runner {
     public:
 };
 
-
+    class cRemoteTempFolder {
-// other utility routines (not specific to service_runner)
-void interactive_ssh(const std::string & server_name, const std::string & command);
-void edit_server(const std::string & server_name);
-void edit_file(const std::string & file_path, const std::string & aftertext);
-
-// check if the service template and config are up to date on the remote server.
-class service_versions {
         public:
-        service_versions(const std::string & server_name, const std::string & service_name);
+            cRemoteTempFolder(const server_env_manager & server_env); // create a temp folder on the remote server
-
+            ~cRemoteTempFolder(); // delete the temp folder on the remote server
-        bool remote_up_to_date() { return remote_template_is_up_to_date() && remote_config_is_up_to_date(); }
+            std::string path() const; // get the path to the temp folder on the remote server
-
-        bool remote_template_is_up_to_date() { return get_version_remote_service_template() == calculate_version_local_service_template(); }
-        bool remote_config_is_up_to_date() { return get_version_remote_config() == calculate_version_local_config(); }    
-
-        XXH64_hash_t calculate_version_local_service_template();
-        XXH64_hash_t calculate_version_local_config();
-
-        void update_stored_remote_versions();
-        XXH64_hash_t get_version_remote_config();
-        XXH64_hash_t get_version_remote_service_template();
-
         private:
-        XXH64_hash_t calculate_version_remote_service_template();
+            std::string mPath;
-        XXH64_hash_t calculate_version_remote_config();
+            const server_env_manager & mServerEnv;
-
-    private:
-        std::string m_server_name;
-        std::string m_service_name;
     };
 
-
 } // namespace dropshell
 
 #endif // SERVICE_RUNNER_HPP

src/service_versions.cpp

@@ -1,70 +0,0 @@
-#include "service_runner.hpp"
-#include "utils/hash.hpp"
-#include "utils/directories.hpp"
-#include "utils/utils.hpp"
-#include "templates.hpp"
-
-#include <fstream>
-#include <filesystem>
-#include <unistd.h>
-
-namespace fs = std::filesystem;
-
-namespace dropshell {
-
-service_versions::service_versions(const std::string & server_name, const std::string & service_name) 
-    : m_server_name(server_name), m_service_name(service_name)
-{
-}
-
-XXH64_hash_t service_versions::calculate_version_local_service_template()
-{
-    template_info tinfo;
-    if (!get_template_info(m_service_name, tinfo)) {
-        return 0;
-    }
-    return hash_directory_recursive(tinfo.local_template_path);
-}
-
-XXH64_hash_t service_versions::calculate_version_local_config()
-{
-    std::string config_path = localpath::service(m_server_name, m_service_name);
-    if (config_path.empty() || !fs::exists(config_path)) {
-        return 0;
-    }
-    return hash_directory_recursive(config_path);
-}
-
-void service_versions::update_stored_remote_versions()
-{
-    XXH64_hash_t template_hash = calculate_version_remote_service_template();
-    XXH64_hash_t config_hash = calculate_version_remote_config();
-
-    // TODO - actually update things!
-}
-
-XXH64_hash_t service_versions::get_version_remote_config()
-{
-    // TODO - actually get the version!
-    return 0;
-}
-
-XXH64_hash_t service_versions::get_version_remote_service_template()
-{
-    // TODO - actually get the version!
-    return 0;
-}
-
-XXH64_hash_t service_versions::calculate_version_remote_service_template()
-{
-    // TODO - actually get the version!
-    return 0;
-}
-
-XXH64_hash_t service_versions::calculate_version_remote_config()
-{   
-    // TODO - actually get the version!
-    return 0;
-}
-
-} // namespace dropshell 

src/services.cpp

@@ -5,6 +5,8 @@
 #include "config.hpp"
 #include "utils/utils.hpp"
 #include "server_env_manager.hpp"
+#include "servers.hpp"
+
 #include <iostream>
 #include <filesystem>
 
@@ -12,90 +14,82 @@ namespace fs = std::filesystem;
 
 namespace dropshell {   
 
-std::vector<ServiceInfo> get_server_services_info(const std::string& server_name) {
+    bool SIvalid(const LocalServiceInfo& service_info) {
-    std::vector<ServiceInfo> services;
+        return !service_info.service_name.empty() && 
+               !service_info.template_name.empty() && 
+               !service_info.local_service_path.empty() && 
+               !service_info.local_template_path.empty();
+    }
+
+std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name) {
+    std::vector<LocalServiceInfo> services;
 
     if (server_name.empty())
         return services;
 
-    std::vector<std::string> local_config_directories = gConfig().get_local_config_directories();
+    std::vector<std::string> local_server_definition_paths = gConfig().get_local_server_definition_paths();
-    if (local_config_directories.empty()) {
+    if (local_server_definition_paths.empty()) {
-        std::cerr << "Error: No local config directories found" << std::endl;
+        std::cerr << "Error: No local server definition paths found" << std::endl;
-        std::cerr << "Run 'dropshell init' to initialise DropShell" << std::endl;
+        std::cerr << "Run 'dropshell edit' to configure DropShell" << std::endl;
         return services;
     }
 
-    for (int i = 0; i < localpath::num_config_directories(); i++) {
+    for (const auto& server_definition_path : local_server_definition_paths) {
-        std::string serverpath = localpath::config_servers(i);
+        fs::path serverpath = server_definition_path + "/" + server_name;
-        if (serverpath.empty()) {
+        if (fs::exists(serverpath)) // service is on that server...
-            std::cerr << "Error: Server directory not found: " << serverpath << std::endl;
+            for (const auto& entry : fs::directory_iterator(serverpath)) {
-            return services;
-        }
-        fs::path server_dir = fs::path(serverpath) / server_name;
-        if (fs::exists(server_dir)) {
-            for (const auto& entry : fs::directory_iterator(server_dir)) {
                 if (fs::is_directory(entry)) {
-                    ServiceInfo service = get_service_info(server_name, entry.path().filename().string());
+                    std::string dirname = entry.path().filename().string();
-                    if (!service.template_name.empty()) {
+                    if (dirname.empty() || dirname[0] == '.' || dirname[0] == '_')
+                        continue;
+                    auto service = get_service_info(server_name, dirname);
+                    if (!service.local_service_path.empty()) 
                         services.push_back(service);
+                    else 
+                        std::cerr << "Warning: Failed to get service info for " << dirname << " on server " << server_name << std::endl;
                 }
+            } // end of for 
     }
-            }
-        } // end of for (int i = 0; i < getNumConfigDirectories(); i++)
-    }
-    
    
     return services;
 }
 
 
-ServiceInfo get_service_info(const std::string &server_name, const std::string &service_name)
+LocalServiceInfo get_service_info(const std::string &server_name, const std::string &service_name)
 {
-    ServiceInfo service;
+    LocalServiceInfo service;
 
     if (server_name.empty() || service_name.empty())
-        return ServiceInfo();
+        return LocalServiceInfo();
 
     service.service_name = service_name;
 
     service.local_service_path = localpath::service(server_name, service_name);
     if (service.local_service_path.empty()) 
-        return ServiceInfo();
+        return LocalServiceInfo();
 
 
     // now set the template name and path.
-    std::string local_service_env_path = localfile::service_env(server_name, service_name);
+    std::map<std::string, std::string> variables;
-    envmanager env(local_service_env_path);
+    if (!get_all_service_env_vars(server_name, service_name, variables))
-    if (!env.load()) {
+        return LocalServiceInfo();
-        if (std::filesystem::exists(localpath::service(server_name, service_name)))
-            std::cerr << "Error: service malformed - service.env missing from " << local_service_env_path << std::endl;
-        else
-        {
-            template_info tinfo;
-            get_template_info(service_name, tinfo);
-            std::string template_name = service_name;
-            if (tinfo.local_template_path.empty())
-                template_name = "TEMPLATE";
-            std::cerr << "Error: you need to create that service first, with:  dropshell create-service " << server_name << " "<<template_name<<" " << service_name << std::endl;
-        }
-        return ServiceInfo();
-    }
-    service.template_name = env.get_variable("TEMPLATE");
  
-    if (service.template_name.empty()) {
+    // confirm TEMPLATE is defined.
-        std::cerr << "Error: TEMPLATE variable not defined in " << local_service_env_path << std::endl;
+    auto it = variables.find("TEMPLATE");
-        return ServiceInfo();
+    if (it == variables.end()) {
+        std::cerr << "Error: TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
+        return LocalServiceInfo();
     }
+    service.template_name = it->second;
 
-    template_info tinfo;
+    template_info tinfo = gTemplateManager().get_template_info(service.template_name);
-    if (!get_template_info(service.template_name, tinfo)) {
+    if (!tinfo.is_set()) {
         std::cerr << "Error: Template '" << service.template_name << "' not found" << std::endl;
-        return ServiceInfo();
+        return LocalServiceInfo();
     }
 
     // find the template path   
-    service.local_template_path = tinfo.local_template_path;
+    service.local_template_path = tinfo.local_template_path();
-    service.local_template_default_env_path = tinfo.local_template_path + "/_default.env";
 
     return service;
 }
@@ -107,7 +101,7 @@ std::set<std::string> get_used_commands(const std::string &server_name, const st
     if (server_name.empty() || service_name.empty())
         return commands;
 
-    ServiceInfo service_info = get_service_info(server_name, service_name);
+    auto service_info = get_service_info(server_name, service_name);
     if (service_info.local_template_path.empty()) {
         std::cerr << "Error: Service not found: " << service_name << std::endl;
         return commands;
@@ -130,13 +124,13 @@ std::set<std::string> list_backups(const std::string &server_name, const std::st
         return backups;
 
     // need to find the template for the service.
-    ServiceInfo service_info = get_service_info(server_name, service_name);
+    auto service_info = get_service_info(server_name, service_name);
     if (service_info.local_template_path.empty()) {
         std::cerr << "Error: Service not found: " << service_name << std::endl;
         return backups;
     }
 
-    std::string backups_dir = localpath::backups_path();
+    std::string backups_dir = gConfig().get_local_backup_path();
     if (backups_dir.empty())
         return backups;
 
@@ -181,8 +175,8 @@ bool create_service(const std::string &server_name, const std::string &template_
         return false;
     }
 
-    template_info tinfo;
+    template_info tinfo = gTemplateManager().get_template_info(template_name);
-    if (!get_template_info(template_name, tinfo))
+    if (!tinfo.is_set())
     {
         if (!silent)
         {
@@ -194,21 +188,95 @@ bool create_service(const std::string &server_name, const std::string &template_
         return false;
     }
 
+    // check template is all good.
+    if (!gTemplateManager().test_template(tinfo.local_template_path()))
+    {
+        if (!silent)
+            std::cerr << "Error: Template '" << template_name << "' is not valid" << std::endl;
+        return false;
+    }
+
     // create the service directory
     fs::create_directory(service_dir);
 
-    // copy the template example files to the service directory
+    // copy the template config files to the service directory
-    recursive_copy(tinfo.local_template_path+"/example", service_dir);
+    recursive_copy(tinfo.local_template_path()/"config", service_dir);
 
     if (!silent)
     {
         std::cout << "Service " << service_name <<" created successfully"<<std::endl;
         std::cout << std::endl;
         std::cout << "To complete the installation, please:" << std::endl;
-        std::cout << "1. edit the service.env file:     dropshell edit " << server_name << " " << service_name << std::endl;
+        std::cout << "1. edit the service config file:  dropshell edit " << server_name << " " << service_name << std::endl;
         std::cout << "2. install the remote service:    dropshell install " << server_name << " " << service_name << std::endl;
     }
     return true;
 }
 
+
+
+bool get_all_service_env_vars(const std::string &server_name, const std::string &service_name, std::map<std::string, std::string> & all_env_vars)
+{
+    all_env_vars.clear();
+
+    if (localpath::service(server_name, service_name).empty() || !fs::exists(localpath::service(server_name, service_name)))
+    {
+        std::cerr << "Error: Service not found: " << service_name << std::endl;
+        return false;
+    }
+
+    // add in some handy variables.
+    all_env_vars["CONFIG_PATH"] = remotepath::service_config(server_name,service_name);
+    all_env_vars["SERVER"] = server_name;
+    all_env_vars["SERVICE"] = service_name;
+    all_env_vars["AGENT_PATH"] = remotepath::service_template(server_name, "dropshell-agent") + "/shared";
+
+    ServerInfo server_info = get_server_info(server_name);
+    if (server_info.ssh_host.empty()) 
+        std::cerr << "Error: Server " << server_name << " not found - ssh_host empty, so HOST_NAME not set" << std::endl;
+    all_env_vars["HOST_NAME"] = server_info.ssh_host;
+
+    // Lambda function to load environment variables from a file
+    auto load_env_file = [&all_env_vars](const std::string& file) {
+        if (!file.empty() && std::filesystem::exists(file)) {
+            std::map<std::string, std::string> env_vars;
+            envmanager env_manager(file);
+            env_manager.load();
+            env_manager.get_all_variables(env_vars);
+            all_env_vars.merge(env_vars);
+        }
+        else
+            std::cout << "Warning: Expected environment file not found: " << file << std::endl;
+    };
+
+    // Load environment files
+    load_env_file(localfile::service_env(server_name, service_name));
+    load_env_file(localfile::template_info_env(server_name, service_name));
+
+    // determine template name.
+    auto it = all_env_vars.find("TEMPLATE");
+    if (it == all_env_vars.end()) {
+        std::cerr << std::endl << std::endl;
+        std::cerr << "Error: TEMPLATE variable not defined in service " << service_name << " on server " << server_name << std::endl;
+        std::cerr << "The TEMPLATE variable is required to determine the template name." << std::endl;
+        std::cerr << "Please check the service.env file and the .template_info.env file in:" << std::endl;
+        std::cerr << "    " << localpath::service(server_name, service_name) << std::endl << std::endl;
+        return false;
+    }
+    template_info tinfo = gTemplateManager().get_template_info(it->second);
+    if (!tinfo.is_set()) {
+        std::cerr << "Error: Template '" << it->second << "' not found" << std::endl;
+        return false;
+    }
+
+    std::string default_env_file = tinfo.local_template_path()/"_default.env";
+    if (!fs::exists(default_env_file)) {
+        std::cerr << "Error: Template default env file '" << default_env_file << "' not found" << std::endl;
+        return false;
+    }
+
+    load_env_file(default_env_file);
+    return true;
+}
+
 } // namespace dropshell

src/services.hpp

@@ -4,21 +4,27 @@
 #include <string>
 #include <vector>
 #include <set>
+#include <map>
 
 namespace dropshell {
 
-    struct ServiceInfo {
+    struct LocalServiceInfo {
         std::string service_name;
         std::string template_name;
         std::string local_service_path;
         std::string local_template_path;
-        std::string local_template_default_env_path;
     };
     
-    std::vector<ServiceInfo> get_server_services_info(const std::string& server_name);
+    bool SIvalid(const LocalServiceInfo& service_info);
-    ServiceInfo get_service_info(const std::string& server_name, const std::string& service_name);
+    
+    std::vector<LocalServiceInfo> get_server_services_info(const std::string& server_name);
+
+    LocalServiceInfo get_service_info(const std::string& server_name, const std::string& service_name);
     std::set<std::string> get_used_commands(const std::string& server_name, const std::string& service_name);
 
+    // get all env vars for a given service
+    bool get_all_service_env_vars(const std::string& server_name, const std::string& service_name, std::map<std::string, std::string> & all_env_vars);
+
     // list all backups for a given service (across all servers)
     std::set<std::string> list_backups(const std::string& server_name, const std::string& service_name);
 

src/templates.cpp

@@ -1,7 +1,3 @@
-    #include "templates.hpp"
-    #include "config.hpp"   
-    #include "utils/directories.hpp"
-    #include "utils/utils.hpp"
     #include <filesystem>
     #include <iostream>
     #include <fstream>
@@ -9,12 +5,23 @@
     #include <string>
     #include <algorithm>
     #include <iomanip>
+    #include <map>
+    #include <libassert/assert.hpp>
+
+    #include "utils/envmanager.hpp"
+    #include "utils/directories.hpp"
+    #include "utils/utils.hpp"
+    #include "templates.hpp"
+    #include "config.hpp"   
 
     namespace dropshell {
 
+    // ------------------------------------------------------------------------------------------------
+    // template_source_local
+    // ------------------------------------------------------------------------------------------------
 
-    bool get_templates(std::vector<template_info>& templates) {
+    std::set<std::string> template_source_local::get_template_list() {
-        templates.clear();
+        std::set<std::string> templates;
             
         // Helper function to add templates from a directory
         auto add_templates_from_dir = [&templates](const std::string& dir_path) {
@@ -22,112 +29,180 @@
                 return;
             
             for (const auto& entry : std::filesystem::directory_iterator(dir_path)) 
-                if (entry.is_directory()) {
+                if (entry.is_directory()) 
-
+                    templates.insert(entry.path().filename().string());
-                    template_info info(entry.path().filename().string(), entry.path().string());
-
-                    // Check if template with same name already exists
-                    bool duplicate = false;
-                    auto it = std::find_if(templates.begin(), templates.end(),
-                        [&info](const template_info& t) { return t.template_name == info.template_name; });
-                    duplicate = (it!=templates.end());
-
-                    if (!duplicate) 
-                        templates.push_back(info);
-                }
         };
 
-        // add templates from the local config directories
+        add_templates_from_dir(mLocalPath);
-        std::vector<std::string> template_config_directories;
+        return templates;
-        get_all_template_config_directories(template_config_directories);
-        for (const auto& path : template_config_directories) {
-            add_templates_from_dir(path);
     }
 
-        return true;
+    bool template_source_local::has_template(const std::string& template_name) {
-    }
+        std::filesystem::path path = mLocalPath / template_name;
-
-
-    bool get_template_info(const std::string& template_name, template_info& info) {
-        // add templates from the local config directories
-        std::vector<std::string> paths_to_search;
-        get_all_template_config_directories(paths_to_search);
-
-        for (const auto& path : paths_to_search) {
-            std::filesystem::path full_path = path + "/" + template_name;
-            if (std::filesystem::exists(full_path)) 
-            {
-                info.template_name = template_name;
-                info.local_template_path = full_path.string();
-                return true;
-            }
-        }
-        std::cout << "Warning: Template '" << template_name << "' not found" << std::endl;
-        return false;
-    }
-
-    bool template_command_exists(const std::string &template_name, const std::string &command)
-    {
-        template_info info;
-        if (!get_template_info(template_name, info)) {
-            return false;
-        }
-        
-        std::string path = info.local_template_path + "/" + command + ".sh";
         return (std::filesystem::exists(path));
     }
 
-    void list_templates() {
+    bool template_source_local::template_command_exists(const std::string& template_name, const std::string& command) {
-        std::vector<template_info> templates;
+        std::filesystem::path path = mLocalPath / template_name / (command+".sh");
-        
+        return std::filesystem::exists(path);
-        if (!get_templates(templates)) {
-            std::cerr << "Error: Failed to get templates" << std::endl;
-            return;
     }    
 
+    template_info template_source_local::get_template_info(const std::string& template_name) {
+        std::filesystem::path path = mLocalPath / template_name;
+
+        if (!std::filesystem::exists(path)) 
+            return template_info();
+
+        return template_info(
+            template_name,
+            mLocalPath.string(),
+            path
+        );
+    }
+
+
+    
+    // ------------------------------------------------------------------------------------------------
+    // template_source_registry
+    // ------------------------------------------------------------------------------------------------
+
+    std::set<std::string> template_source_registry::get_template_list()
+    {
+        #pragma message("TODO: Implement template_source_registry::get_template_list")
+        return std::set<std::string>();
+    }
+
+    bool template_source_registry::has_template(const std::string& template_name)
+    {
+        #pragma message("TODO: Implement template_source_registry::has_template")
+        return false;
+    }
+
+    template_info template_source_registry::get_template_info(const std::string& template_name)
+    {
+        #pragma message("TODO: Implement template_source_registry::get_template_info")
+        return template_info();
+    }
+
+    bool template_source_registry::template_command_exists(const std::string& template_name, const std::string& command)
+    {
+        #pragma message("TODO: Implement template_source_registry::template_command_exists")
+        return false;
+    }
+
+    std::filesystem::path template_source_registry::get_cache_dir()
+    {
+        #pragma message("TODO: Implement template_source_registry::get_cache_dir")
+        return std::filesystem::path();
+    }
+
+
+
+
+
+    // ------------------------------------------------------------------------------------------------
+    // template_manager
+    // ------------------------------------------------------------------------------------------------
+
+    void template_manager::list_templates() const {
+        ASSERT(mLoaded && mSources.size() > 0);
+        auto templates = get_template_list();
+        
         if (templates.empty()) {
             std::cout << "No templates found." << std::endl;
             return;
         }
         
         std::cout << "Available templates:" << std::endl;
-        std::cout << std::left << std::setw(20) << "Name" << "Path" << std::endl;
+
+        // print templates.
         std::cout << std::string(60, '-') << std::endl;        
-        
+        bool first = true;
         for (const auto& t : templates) {
-            std::cout << std::left << std::setw(20) << t.template_name << t.local_template_path << std::endl;
+            std::cout << (first ? "" : ", ") << t;
+            first = false;
         }
+        std::cout << std::endl;
+        std::cout << std::string(60, '-') << std::endl;
     }
 
-    void create_template(const std::string& template_name) {
+    std::set<std::string> template_manager::get_template_list() const
+    {
+        ASSERT(mLoaded && mSources.size() > 0);
+        std::set<std::string> templates;
+        for (const auto& source : mSources) {
+            auto source_templates = source->get_template_list();
+            templates.insert(source_templates.begin(), source_templates.end());
+        }
+        return templates;
+    }
+
+    bool template_manager::has_template(const std::string &template_name) const
+    {
+        ASSERT(mLoaded && mSources.size() > 0);
+        template_source_interface* source = get_source(template_name);
+        if (!source) 
+            return false;
+        return true;
+    }
+
+    template_info template_manager::get_template_info(const std::string &template_name) const
+    {
+        ASSERT(mLoaded && mSources.size() > 0);
+        template_source_interface* source = get_source(template_name);
+        if (source)
+            return source->get_template_info(template_name);
+
+        // fail
+        return template_info();
+}
+
+    bool template_manager::template_command_exists(const std::string &template_name, const std::string &command) const
+    {
+        ASSERT(mLoaded && mSources.size() > 0);
+        template_source_interface* source = get_source(template_name);
+        if (!source) {
+            std::cerr << "Error: Template '" << template_name << "' not found" << std::endl;
+            return false;
+        }
+        return source->template_command_exists(template_name, command);
+    }
+
+    bool template_manager::create_template(const std::string &template_name) const
+    {
         // 1. Create a new directory in the user templates directory
-        std::vector<std::string> local_config_directories = gConfig().get_local_config_directories();
+        std::vector<std::string> local_server_definition_paths = gConfig().get_local_server_definition_paths();
         
-        if (local_config_directories.empty()) {
+        if (local_server_definition_paths.empty()) {
-            std::cerr << "Error: No local config directories found" << std::endl;
+            std::cerr << "Error: No local server definition paths found" << std::endl;
-            std::cerr << "Run 'dropshell init' to initialise DropShell" << std::endl;
+            std::cerr << "Run 'dropshell edit' to configure DropShell" << std::endl;
-            return;
+            return false;
         }
 
-        template_info info;
+        auto info = get_template_info(template_name);
-        if (get_template_info(template_name, info)) {
+        if (info.is_set()) {
-            std::cerr << "Error: Template '" << template_name << "' already exists at " << info.local_template_path << std::endl;
+            std::cerr << "Error: Template '" << template_name << "' already exists at " << info.locationID() << std::endl;
-            return;
+            return false;
         }
 
-        std::string new_template_path = localpath::config_templates() + "/" + template_name;
+        auto local_template_paths = gConfig().get_template_local_paths();
+        if (local_template_paths.empty()) {
+            std::cerr << "Error: No local template paths found" << std::endl;
+            std::cerr << "Run 'dropshell edit' to add one to the DropShell config" << std::endl;
+            return false;
+        }
+        std::string new_template_path = local_template_paths[0] + "/" + template_name;
         
         // Create the new template directory
         std::filesystem::create_directories(new_template_path);
         
         // 2. Copy the example template from the system templates directory
-        std::string system_templates_dir = localpath::system_templates();
+        auto example_info = gTemplateManager().get_template_info("example-nginx");
-        std::string example_template_path = system_templates_dir + "/example-nginx";
+        if (!example_info.is_set()) {
-        
+            std::cerr << "Error: Example template not found" << std::endl;
-        if (!std::filesystem::exists(example_template_path)) {
+            return false;
-            std::cerr << "Error: Example template not found at " << example_template_path << std::endl;
-            return;
         }
+        std::string example_template_path = example_info.local_template_path();
                 
         // Copy all files from example template to new template
         for (const auto& entry : std::filesystem::recursive_directory_iterator(example_template_path)) {
@@ -141,14 +216,13 @@
             }
         }
 
-        // modify the TEMPLATE=example line in the service.env file to TEMPLATE=<template_name>
+        // modify the TEMPLATE=example line in the .template_info.env file to TEMPLATE=<template_name>
         std::string search_string = "TEMPLATE=";
         std::string replacement_line = "TEMPLATE=" + template_name;
-        // replace the line in the example/service.env file with the replacement line
+        std::string service_env_path = new_template_path + "/config/.template_info.env";
-        std::string service_env_path = new_template_path + "/example/service.env";
         if (!replace_line_in_file(service_env_path, search_string, replacement_line)) {
-            std::cerr << "Error: Failed to replace TEMPLATE= line in service.env file" << std::endl;
+            std::cerr << "Error: Failed to replace TEMPLATE= line in the .template_info.env file" << std::endl;
-            return;
+            return false;
         }
 
         // 3. Print out the README.txt file and the path
@@ -166,27 +240,120 @@
                 readme_file.close();
             }
             std::cout << std::string(60, '-') << std::endl;
-        } else {
-            std::cout << "No README.txt file found in the template." << std::endl;
         } 
 
         std::cout << std::endl;
         std::cout << "Template '" << template_name << "' created at " << new_template_path << std::endl;
+        return test_template(new_template_path);
     }
 
-    bool get_all_template_config_directories(std::vector<std::string> &template_config_directories)
+    void template_manager::load_sources()
     {
-        template_config_directories.clear();
+        ASSERT(mSources.empty());
-        for (int i = 0; i < localpath::num_config_directories(); i++) {
+        ASSERT(gConfig().is_config_set());
-            std::string config_templates_path = localpath::config_templates(i);
+        ASSERT(!mLoaded);
-            if (config_templates_path.empty()) {
+        auto local_template_paths = gConfig().get_template_local_paths();
-                std::cerr << "Error: Templates directory not found: " << config_templates_path << std::endl;
+        if (local_template_paths.empty())
+            return;
+        for (const auto& path : local_template_paths) 
+            mSources.push_back(std::make_unique<template_source_local>(path));
+
+        auto registry_urls = gConfig().get_template_registry_urls();
+        for (const auto& url : registry_urls) 
+            mSources.push_back(std::make_unique<template_source_registry>(url));
+
+        mLoaded = true;
+    }
+
+    void template_manager::print_sources() const
+    {
+        std::cout << "Template sources: ";
+        for (const auto& source : mSources) {
+            std::cout << "[" << source->get_description() << "] ";
+        }
+        std::cout << std::endl;
+    }
+
+    bool template_manager::required_file(std::string path, std::string template_name)
+    {
+        if (!std::filesystem::exists(path)) {
+            std::cerr << "Error: " << path << " file not found in template - REQUIRED." << template_name << std::endl;
             return false;
         }
-            template_config_directories.push_back(config_templates_path);
-        }
-        template_config_directories.push_back(localpath::system_templates());
         return true;
     }
 
+    template_source_interface *template_manager::get_source(const std::string &template_name) const
+    {
+        ASSERT(mLoaded && mSources.size() > 0);
+        for (const auto& source : mSources) {
+            if (source->has_template(template_name)) {
+                return source.get();
+            }
+        }
+        return nullptr;
+    }
+
+    bool template_manager::test_template(const std::string &template_path)
+    {
+        std::string template_name = std::filesystem::path(template_path).filename().string();
+
+        std::vector<std::string> required_files = {
+            "config/service.env",
+            "config/.template_info.env",
+            "_default.env",
+            "install.sh",
+            "uninstall.sh",
+            "nuke.sh"
+        };
+
+        for (const auto& file : required_files) {
+            if (!required_file(template_path + "/" + file, template_name)) 
+                return false;
+        }        
+
+        // ------------------------------------------------------------
+        // check TEMPLATE= line.
+        std::map<std::string, std::string> all_env_vars;
+        std::vector<std::string> env_files = {
+            "config/service.env",
+            "config/.template_info.env"
+        };
+        for (const auto& file : env_files) {
+            { // load service.env from the service on this machine.
+            std::map<std::string, std::string> env_vars;
+            envmanager env_manager(template_path + "/" + file);
+            env_manager.load();
+            env_manager.get_all_variables(env_vars);
+            all_env_vars.merge(env_vars);
+            }
+        }
+
+        // determine template name.
+        auto it = all_env_vars.find("TEMPLATE");
+        if (it == all_env_vars.end()) {
+            std::cerr << "Error: TEMPLATE variable not found in " << template_path << std::endl;
+            return false;
+        }
+
+        std::string env_template_name = it->second;
+        if (env_template_name.empty()) {
+            std::cerr << "Error: TEMPLATE variable is empty in " << template_path << std::endl;
+            return false;
+        }
+
+        if (env_template_name != template_name) {
+            std::cerr << "Error: TEMPLATE variable is wrong in " << template_path << std::endl;
+            return false;
+        }
+
+        return true;
+    }
+
+    template_manager & gTemplateManager()
+    {
+        static template_manager instance;
+        return instance;
+    }
+
     } // namespace dropshell

src/templates.hpp

@@ -1,36 +1,110 @@
-
 #include <string>
 #include <vector>
 #include <filesystem>
+#include <memory>
+#include <set>
+
+#include "utils/json.hpp"
 
 namespace dropshell {
 
+typedef enum template_source_type {
+    TEMPLATE_SOURCE_TYPE_LOCAL,
+    TEMPLATE_SOURCE_TYPE_REGISTRY,
+    TEMPLATE_SOURCE_NOT_SET
+} template_source_type;
+
 class template_info {
     public:
-        template_info() {}
+        template_info() : mIsSet(false) {}
-        template_info(std::string n, std::string p) : template_name(n), local_template_path(p) {}
+        template_info(const std::string& template_name, const std::string& location_id, const std::filesystem::path& local_template_path) : mTemplateName(template_name), mLocationID(location_id), mTemplateLocalPath(local_template_path), mIsSet(true) {}
-
+        virtual ~template_info() {}
-        std::string template_name;
+        bool is_set() { return mIsSet; }
-        std::string local_template_path;
+        std::string name() { return mTemplateName; }
+        std::string locationID() { return mLocationID; }
+        std::filesystem::path local_template_path() { return mTemplateLocalPath; }
+    private:
+        std::string mTemplateName;
+        std::string mLocationID;
+        std::filesystem::path mTemplateLocalPath; // source or cache.
+        bool mIsSet;
 };
 
-    // templates are stored in multiple locations:
+class template_source_interface {
-    // 1. /opt/dropshell/templates
+    public:
-    // 2. CONFIG_DIR/templates 
+        virtual ~template_source_interface() {}
+        virtual std::set<std::string> get_template_list() = 0;
+        virtual bool has_template(const std::string& template_name) = 0;
+        virtual template_info get_template_info(const std::string& template_name) = 0;
+        virtual bool template_command_exists(const std::string& template_name,const std::string& command) = 0;
 
+        virtual std::string get_description() = 0;
+};
 
-    bool get_templates(std::vector<template_info>& templates);
+class template_source_registry : public template_source_interface {
-    bool get_template_info(const std::string& template_name, template_info& info);        
+    public:
+        template_source_registry(std::string URL) : mURL(URL) {}
+
+        ~template_source_registry() {}
+
+        std::set<std::string> get_template_list();
+        bool has_template(const std::string& template_name);
+        template_info get_template_info(const std::string& template_name);
         bool template_command_exists(const std::string& template_name,const std::string& command);
-    void list_templates();
 
+        std::string get_description() { return "Registry: " + mURL; }
+    private:
+        std::filesystem::path get_cache_dir();
+    private:
+        std::string mURL;
+        std::vector<nlohmann::json> mTemplates; // cached list.
+};
 
-    // create a template
+class template_source_local : public template_source_interface {
-    // 1. create a new directory in the user templates directory
+    public:
-    // 2. copy the example template from the system templates directory into the new directory
+        template_source_local(std::string local_path) : mLocalPath(local_path) {}
-    // 3. print out the README.txt file in the new template directory, and the path to the new template
+        ~template_source_local() {}
-    void create_template(const std::string& template_name);
+        std::set<std::string> get_template_list();
+        bool has_template(const std::string& template_name);
+        template_info get_template_info(const std::string& template_name);
+        bool template_command_exists(const std::string& template_name,const std::string& command);
+
+        std::string get_description() { return "Local: " + mLocalPath.string(); }
+    private:
+        std::filesystem::path mLocalPath;
+};
+
+class template_manager {
+    public:
+        template_manager() : mLoaded(false) {}
+        ~template_manager() {}
+
+        std::set<std::string> get_template_list() const;
+        bool has_template(const std::string& template_name) const;
+        template_info get_template_info(const std::string& template_name) const;
+
+        bool template_command_exists(const std::string& template_name,const std::string& command) const;
+        bool create_template(const std::string& template_name) const;
+        static bool test_template(const std::string& template_path);
+
+        void list_templates() const;
+
+        void load_sources();
+        void print_sources() const;
+
+        bool is_loaded() const { return mLoaded; }
+        int get_source_count() const { return mSources.size(); }
+
+    private:
+        static bool required_file(std::string path, std::string template_name);
+        template_source_interface* get_source(const std::string& template_name) const;
+
+    private:
+        bool mLoaded;
+        mutable std::vector<std::unique_ptr<template_source_interface>> mSources;
+};
+
+template_manager & gTemplateManager();
 
-    bool get_all_template_config_directories(std::vector<std::string>& template_config_directories);
 
 } // namespace dropshell

src/utils/assert.hpp

@@ -1,58 +0,0 @@
-#ifndef DROPSHELL_ASSERT_HPP
-#define DROPSHELL_ASSERT_HPP
-
-#include <iostream>
-#include <string_view>
-#include <cstdlib> // For std::exit and EXIT_FAILURE
-
-namespace ds {
-
-struct SourceLocation {
-    const char* file_name;
-    int line;
-    const char* function_name;
-};
-
-// Helper macro to create a SourceLocation with current context
-#define DS_CURRENT_LOCATION ds::SourceLocation{__FILE__, __LINE__, __func__}
-
-[[noreturn]] inline void assert_fail(
-    const char* expression,
-    const SourceLocation& location,
-    const char* message = nullptr) {
-    
-    std::cerr << "\033[1;31mAssertion failed!\033[0m\n"
-              << "Expression: \033[1;33m" << expression << "\033[0m\n"
-              << "Location:   \033[1;36m" << location.file_name << ":" 
-                                           << location.line << "\033[0m\n"
-              << "Function:   \033[1;36m" << location.function_name << "\033[0m\n";
-              
-    if (message) {
-        std::cerr << "Message:    \033[1;37m" << message << "\033[0m\n";
-    }
-    
-    std::cerr << std::endl;
-    
-    // Exit the program without creating a core dump
-    std::exit(EXIT_FAILURE);
-}
-
-} // namespace ds
-
-// Standard assertion
-#define ASSERT(condition) \
-    do { \
-        if (!(condition)) { \
-            ds::assert_fail(#condition, DS_CURRENT_LOCATION); \
-        } \
-    } while (false)
-    
-// Assertion with custom message
-#define ASSERT_MSG(condition, message) \
-    do { \
-        if (!(condition)) { \
-            ds::assert_fail(#condition, DS_CURRENT_LOCATION, message); \
-        } \
-    } while (false)
-
-#endif // DROPSHELL_ASSERT_HPP

src/utils/directories.cpp

@@ -11,40 +11,31 @@ namespace dropshell {
 
 
 namespace localfile {
-    std::string dropshell_env() {
+
-    // Try ~/.config/dropshell/dropshell.env
+    std::string dropshell_json() {
-    const char* home = std::getenv("HOME");
+        // Try ~/.config/dropshell/dropshell.json
-    if (home) {
+        std::string homedir = localpath::current_user_home();
-        fs::path user_path = fs::path(home) / ".config" / "dropshell" /  "dropshell.env";
+        if (!homedir.empty()) {
+            fs::path user_path = fs::path(homedir) / ".config" / "dropshell" /  "dropshell.json";
             return user_path.string();
         }
-    std::cerr << "Warning: Couldn't determine user directory" << std::endl;
         return std::string();
     }
 
-    std::string server_env(const std::string &server_name) {
+    std::string server_json(const std::string &server_name) {
-        if (server_name.empty())
-            return std::string();
         std::string serverpath = localpath::server(server_name);
-        if (serverpath.empty()) 
+        return (serverpath.empty() ? "" : (fs::path(serverpath) / "server.json").string());
-            return std::string();
-        return (fs::path(serverpath) / "server.env").string();
     }
 
     std::string service_env(const std::string &server_name, const std::string &service_name) {
-        if (server_name.empty() || service_name.empty())
-            return std::string();
         std::string servicepath = localpath::service(server_name, service_name);
-        if (servicepath.empty())
+        return (servicepath.empty() ? "" : (fs::path(servicepath) / "service.env").string());
-            return std::string();
-        return (fs::path(servicepath) / "service.env").string();
     }
 
-    std::string service_hash(const std::string &server_name, const std::string &service_name) {
+    std::string template_info_env(const std::string &server_name, const std::string &service_name)
-        std::string config_path = localpath::config();
+    {
-        if (server_name.empty() || service_name.empty() || config_path.empty())
+        std::string servicepath = localpath::service(server_name, service_name);
-            return std::string();
+        return (servicepath.empty() ? "" : (fs::path(servicepath) / ".template_info.env").string());
-        return (fs::path(config_path) / ".remote_versions" / server_name / (service_name + ".hash.env")).string();
     }
 
 } // namespace localfile
@@ -53,46 +44,40 @@ namespace localfile {
 // ------------------------------------------------------------------------------------------
 
 namespace localpath {
-        // /opt/dropshell/templates
-        std::string system_templates() {
-            return "/opt/dropshell/templates";
-        }
-
-        // configured by user - defaults to first config_path/backups.
-        std::string backups_path() {
-            return gConfig().get_local_backup_path();
-        }
-
-        int num_config_directories() {
-            return gConfig().get_local_config_directories().size();
-        }
-
-        std::string config(int index) {
-            return (num_config_directories()>index) ? gConfig().get_local_config_directories()[index] : "";
-        }
-
-        std::string config_templates(int index) {
-            return (num_config_directories()>index) ? (gConfig().get_local_config_directories()[index]+"/templates") : "";
-        }
-
-        std::string config_servers(int index) {
-            return (num_config_directories()>index) ? (gConfig().get_local_config_directories()[index]+"/servers") : "";
-        }
-
         std::string server(const std::string &server_name) {
-            for (auto &dir : gConfig().get_local_config_directories()) 
+            for (std::filesystem::path dir : gConfig().get_local_server_definition_paths()) 
-                if (fs::exists(dir + "/servers/" + server_name))
+                if (fs::exists(dir / server_name))
-                    return dir + "/servers/" + server_name;
+                    return dir / server_name;
+
             return "";
         }
 
         std::string service(const std::string &server_name, const std::string &service_name) {
             std::string serverpath = localpath::server(server_name);
-            return (serverpath.empty() ? "" : (serverpath+"/"+service_name));
+            return ((serverpath.empty() || service_name.empty()) ? "" : (serverpath+"/"+service_name));
+        }
+
+        std::string remote_versions(const std::string &server_name, const std::string &service_name)
+        {
+            std::string template_cache_path = gConfig().get_local_template_cache_path();
+            return ((template_cache_path.empty() || service_name.empty()) ? "" : 
+               (template_cache_path+"/remote_versions/"+service_name+".json"));
+        }
+        std::string local_bin(){
+            return current_user_home() + "/.local/bin";
+        }
+        std::string current_user_home(){
+            char * homedir = std::getenv("HOME");
+            if (homedir)
+            {
+                std::filesystem::path homedir_path(homedir);
+                return fs::canonical(homedir_path).string();
+            }
+            std::cerr << "Warning: Couldn't determine user directory" << std::endl;
+            return std::string();
         }
 } // namespace localpath
 
-
 // ------------------------------------------------------------------------------------------
 
     // remote paths
@@ -151,6 +136,12 @@ namespace remotepath {
         return (dsp.empty() ? "" : (dsp + "/backups"));
     }
 
+    std::string temp_files(const std::string &server_name)
+    {
+        std::string dsp = DROPSHELL_DIR(server_name);
+        return (dsp.empty() ? "" : (dsp + "/temp_files"));
+    }
+
     std::string service_env(const std::string &server_name, const std::string &service_name) 
     {
         std::string service_path = service_config(server_name, service_name);
@@ -162,11 +153,18 @@ namespace remotepath {
 // ------------------------------------------------------------------------------------------
 // Utility functions
 
-std::string get_parent(std::string path)
+std::string get_parent(const std::filesystem::path path)
 {
     if (path.empty())
         return std::string();
-    return fs::path(path).parent_path().string();
+    return path.parent_path().string();
+}
+
+std::string get_child(const std::filesystem::path path)
+{
+    if (path.empty())
+        return std::string();
+    return path.filename().string();
 }
 
 } // namespace dropshell

src/utils/directories.hpp

@@ -2,6 +2,7 @@
 #define DIRECTORIES_HPP
 
 #include <string>
+#include <filesystem>
 
 namespace dropshell {
 
@@ -9,45 +10,52 @@ namespace dropshell {
 
     //------------------------------------------------------------------------------------------------
     // local user config directories
-    // config_path
+    
-    //   |-- servers
+    // ~/.config/dropshell/dropshell.json
-    //   |   |-- server_name
+    
-    //   |       |-- server.env
+    // server_definition_path
-    //   |       |-- services
+    //    |-- <server_name>
-    //   |           |-- service_name
+    //        |-- server.json
-    //   |               |-- service.env
+    //        |-- services
-    //   |               |-- (other config files for specific server&service)
+    //            |-- <service_name>
+    //                |-- service.env
+    //                |-- .template_info.env
+    //                |-- (...other config files for specific server&service...)
+    
+    // backup path
+    //    |-- katie-_-squashkiwi-_-squashkiwi-test-_-2025-04-28_21-23-59.tgz
+ 
+    // temp files path
+
+    // template cache path
     //    |-- templates
-    //   |   |-- template_name
+    //    |   |-- <template_name>.json
-    //   |       |-- (script files)
+    //    |   |-- <template_name>
-    //   |       |-- example
+    //    |       |-- (...script files...)
+    //    |       |-- _default.env
+    //    |       |-- config
     //    |           |-- service.env
-    //   |           |-- (other service config files)
+    //    |           |-- .template_info.env
-    //   |-- .remote_versions
+    //    |           |-- (...other service config files...)
-    //   |   |-- server_name
+    //    |-- remote_versions
-    //   |       |-- service_name.hash.env
+    //    |   |-- server_name-service_name.json
+
     namespace localfile {
-        // ~/.config/dropshell/dropshell.conf
+        // ~/.config/dropshell/dropshell.json
-        std::string dropshell_env();
+        std::string dropshell_json();
-        std::string server_env(const std::string &server_name);
+        std::string server_json(const std::string &server_name);
         std::string service_env(const std::string &server_name, const std::string &service_name);
-        std::string service_hash(const std::string &server_name, const std::string &service_name);
+        std::string template_info_env(const std::string &server_name, const std::string &service_name);
     } // namespace localfile
 
     namespace localpath {
-        // /opt/dropshell/templates
-        std::string system_templates();
-
-        // configured by user - defaults to first config_path/backups.
-        std::string backups_path();
-
-        int num_config_directories();
-        std::string config(int index=0);
-        std::string config_templates(int index=0);
-        std::string config_servers(int index=0);
-
         std::string server(const std::string &server_name);
         std::string service(const std::string &server_name, const std::string &service_name);
+
+        std::string remote_versions(const std::string &server_name, const std::string &service_name);
+
+        std::string local_bin();
+        std::string current_user_home();
     } // namespace local
 
 
@@ -55,13 +63,14 @@ namespace dropshell {
     // remote paths
     // DROPSHELL_DIR
     //   |-- backups
+    //   |-- temp_files
     //   |-- services
     //       |-- service name
     //           |-- config
     //               |-- service.env
     //           |-- template
     //               |-- (script files)
-    //               |-- example
+    //               |-- config
     //                   |-- service.env
     //                   |-- (other config files for specific server&service)
 
@@ -76,12 +85,14 @@ namespace dropshell {
         std::string service_config(const std::string &server_name, const std::string &service_name);
         std::string service_template(const std::string &server_name, const std::string &service_name);
         std::string backups(const std::string &server_name);
+        std::string temp_files(const std::string &server_name);
     } // namespace remotepath
 
     //------------------------------------------------------------------------------------------------
     // utility functions
-    std::string get_parent(std::string path);
+    std::string get_parent(const std::filesystem::path path);
-
+    std::string get_child(const std::filesystem::path path);
 } // namespace dropshell
 
-#endif
+
+#endif // DIRECTORIES_HPP

src/utils/envmanager.cpp

@@ -72,18 +72,6 @@ void envmanager::get_all_variables(std::map<std::string, std::string>& variables
     variables = m_variables;
 }
 
-std::string envmanager::get_variable_substituted(std::string key) const {
-    std::string value = get_variable(key);
-    return expand_patterns(value);
-}
-
-void envmanager::get_all_variables_substituted(std::map<std::string, std::string>& variables) const {
-    variables.clear();
-    for (const auto& pair : m_variables) {
-        variables[pair.first] = expand_patterns(pair.second);
-    }
-}
-
 void envmanager::add_variables(std::map<std::string, std::string> variables) {
     for (auto& pair : variables) {
         set_variable(pair.first, pair.second);
@@ -98,26 +86,4 @@ void envmanager::clear_variables() {
     m_variables.clear();
 }
 
-std::string envmanager::expand_patterns(std::string str) const {
-    // Combined regex pattern for both ${var} and $var formats
-    std::regex var_pattern("\\$(?:\\{([^}]+)\\}|([a-zA-Z0-9_]+))");
-    std::string result = str;
-    std::smatch match;
-    
-    while (std::regex_search(result, match, var_pattern)) {
-        // match[1] will contain capture from ${var} format
-        // match[2] will contain capture from $var format
-        std::string var_name = match[1].matched ? match[1].str() : match[2].str();
-        
-        // Get value from system environment variables
-        const char* env_value = std::getenv(var_name.c_str());
-        std::string value = env_value ? env_value : "";
-        
-        result = result.replace(match.position(), match.length(), value);
-    }
-
-    // dequote the result
-    return result;
-}
-
 } // namespace dropshell

src/utils/envmanager.hpp

@@ -25,20 +25,12 @@ class envmanager {
         std::string get_variable(std::string key) const;
         void get_all_variables(std::map<std::string, std::string>& variables) const;
 
-        // get variables, but replace patterns ${var} and $var with the actual environment variable in the returned string.
-        // trim whitespace from the values.
-        std::string get_variable_substituted(std::string key) const;
-        void get_all_variables_substituted(std::map<std::string, std::string>& variables) const;
-
         // add variables to the environment files.
         // trim whitespace from the values.
         void add_variables(std::map<std::string, std::string> variables);
         void set_variable(std::string key, std::string value);
         void clear_variables();
 
-    private:
-        std::string expand_patterns(std::string str) const;
-
     private:
         std::string m_path;
         std::map<std::string, std::string> m_variables;

src/utils/execute.cpp

@@ -0,0 +1,175 @@
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <vector>
+#include <iostream>
+#include <string>
+#include <cstdlib>
+#include <sstream>
+#include <libassert/assert.hpp>
+#include <algorithm>
+#include "execute.hpp"
+#include "contrib/base64.hpp"
+#include "utils/utils.hpp"
+#include "utils/directories.hpp"
+#include <termios.h>
+
+bool EXITSTATUSCHECK(int ret) {
+    return (ret != -1 && WIFEXITED(ret) && (WEXITSTATUS(ret) == 0)); // ret is -1 if the command failed to execute.
+}
+
+namespace dropshell {
+bool __execute_command(std::vector<std::string> command, std::string * output)
+{
+    // Save TTY state if possible
+    struct termios orig_termios;
+    int tty_fd = isatty(STDIN_FILENO) ? STDIN_FILENO : -1;
+    if (tty_fd != -1) {
+        tcgetattr(tty_fd, &orig_termios);
+    }
+
+    std::cout << "Executing command: ";
+    for (auto & x : command) std::cout << "[" << x << "] ";
+    std::cout << std::endl << std::flush;
+
+    int pipefd[2];
+    bool capture = (output != nullptr);
+    if (capture && pipe(pipefd) == -1) {
+        perror("pipe failed");
+        return false;
+    }
+
+    pid_t pid = fork();
+
+    if (pid == -1) {
+        // Fork failed
+        perror("fork failed");
+        return false;
+    } else if (pid == 0) {
+        // Child process
+        if (capture) {
+            close(pipefd[0]); // Close read end
+            dup2(pipefd[1], STDOUT_FILENO); // Redirect stdout to pipe
+            dup2(pipefd[1], STDERR_FILENO); // Redirect stderr to pipe
+            close(pipefd[1]);
+        }
+
+        std::vector<const char *> commandvec;
+        for (auto & x : command) {
+            commandvec.push_back(x.c_str());
+        }
+        commandvec.push_back(NULL);
+
+        // if (!silent) {
+        //     std::cout << "Executing command: ";
+        //     for (auto & x : commandvec) std::cout << x << " ";
+        //     std::cout << std::endl;
+        // }
+
+        execvp(commandvec[0], const_cast<char* const*>(commandvec.data()));
+        // If execvp returns, it means an error occurred
+        perror("execvp failed");
+        exit(EXIT_FAILURE); // Exit child process on error
+    } else {
+        // Parent process
+        if (capture) {
+            close(pipefd[1]); // Close write end
+            char buffer[256];
+            ssize_t count;
+            while ((count = read(pipefd[0], buffer, sizeof(buffer))) > 0) {
+                output->append(buffer, count);
+            }
+            close(pipefd[0]);
+        }
+        int ret;
+        // Wait for the child process to complete
+        waitpid(pid, &ret, 0);
+
+        // Restore TTY state if possible
+        if (tty_fd != -1) {
+            tcsetattr(tty_fd, TCSANOW, &orig_termios);
+        }
+
+        return EXITSTATUSCHECK(ret);
+    }
+}
+
+bool execute_command(const sSSHInfo * ssh_info, const sCommand command, cMode mode, std::string * output)
+{
+    if (command.get_command_to_run().empty())
+        return false;
+
+    std::vector<std::string> commandvec;
+    
+    // Construct the shell command with proper environment variables and directory
+    std::string shell_command;
+    
+    // Add working directory if provided
+    if (!command.get_directory_to_run_in().empty()) {
+        shell_command += "cd " + quote(command.get_directory_to_run_in()) + " && ";
+    }
+    
+    // Add environment variables if provided
+    for (const auto& env_var : command.get_env_vars()) {
+        shell_command += env_var.first + "=" + quote(env_var.second) + " ";
+    }
+    
+    // Add the command arguments properly joined
+    const auto& args = command.get_command_to_run();
+    std::ostringstream cmd_stream;
+    for (size_t i = 0; i < args.size(); ++i) {
+        if (i > 0) cmd_stream << " ";
+        cmd_stream << args[i];
+    }
+    shell_command += cmd_stream.str();
+    
+    if (ssh_info) {
+        // Use bash -c for SSH to ensure proper command execution
+        commandvec = {
+            "/usr/bin/ssh", 
+            "-p", ssh_info->port, 
+            (hasFlag(mode, cMode::CaptureOutput) ? "" : "-tt"), 
+            ssh_info->user + "@" + ssh_info->host,
+            "bash", "-c", halfquote(shell_command)
+        };
+    } else {
+        // Local execution
+        commandvec = {"bash", "-c", halfquote(shell_command)};
+    }
+    
+    // Remove any empty strings
+    commandvec.erase(
+        std::remove_if(commandvec.begin(), commandvec.end(), 
+                      [](const std::string& s) { return s.empty(); }),
+        commandvec.end());
+    
+    if (hasFlag(mode, cMode::CaptureOutput)) {
+        ASSERT(output, "Capture output mode requires an output string to be provided");
+        return __execute_command(commandvec, output);
+    } else {
+        return __execute_command(commandvec, nullptr);
+    }
+}
+
+bool execute_command(const sSSHInfo ssh_info, const sCommand command, const cMode mode)
+{
+    return execute_command(&ssh_info, command, mode, nullptr);
+}
+
+bool execute_command(const sSSHInfo ssh_info, const sCommand command, const cMode mode, std::string & output)
+{
+    return execute_command(&ssh_info, command, mode, &output);
+}
+
+bool execute_command(const sCommand command, const cMode mode)
+{
+    return execute_command(nullptr, command, mode, nullptr);
+}
+
+bool execute_command(const sCommand command, const cMode mode, std::string & output)
+{
+    return execute_command(nullptr, command, mode, &output);
+}
+
+
+} // namespace dropshell

src/utils/execute.hpp

@@ -0,0 +1,70 @@
+#ifndef EXECUTE_HPP
+#define EXECUTE_HPP
+
+#include <string>
+#include <map>
+#include <optional>
+
+namespace dropshell {
+
+class sCommand;
+
+// mode bitset
+enum class cMode {
+    Defaults = 0,
+    Silent = 1,
+    CaptureOutput = 2
+};
+
+inline cMode operator&(cMode lhs, cMode rhs) {return static_cast<cMode>(static_cast<int>(lhs) & static_cast<int>(rhs));}
+inline cMode operator+(cMode lhs, cMode rhs) {return static_cast<cMode>(static_cast<int>(lhs) | static_cast<int>(rhs));}
+inline cMode operator-(cMode lhs, cMode rhs) {return static_cast<cMode>(static_cast<int>(lhs) & ~static_cast<int>(rhs));}
+inline cMode operator|(cMode lhs, cMode rhs) {return static_cast<cMode>(static_cast<int>(lhs) | static_cast<int>(rhs));}
+inline cMode operator|=(cMode & lhs, cMode rhs) {return lhs = lhs | rhs;}
+inline bool hasFlag(cMode mode, cMode flag) {return (mode & flag) == flag;}
+
+
+typedef struct sSSHInfo {
+    std::string host;
+    std::string user;
+    std::string port;
+} sSSHInfo;
+
+bool execute_command(const sSSHInfo ssh_info, const sCommand command, const cMode mode);
+bool execute_command(const sSSHInfo ssh_info, const sCommand command, const cMode mode, std::string & output);
+bool execute_command(const sCommand command, const cMode mode);
+bool execute_command(const sCommand command, const cMode mode, std::string & output);
+
+// ------------------------------------------------------------------------------------------------
+
+
+
+// class to hold a command to run on the remote server.
+class sCommand {
+    public:
+        sCommand(std::string directory_to_run_in, const std::vector<std::string> & command_to_run, const std::map<std::string, std::string> & env_vars) : 
+            mDir(directory_to_run_in), mCmdArgs(command_to_run), mEnvVars(env_vars) {}
+        sCommand(const std::vector<std::string> & command_to_run) : 
+            mDir(""), mCmdArgs(command_to_run), mEnvVars({}) {}
+        sCommand() : mDir(""), mCmdArgs({}), mEnvVars({}) {}
+
+        std::string get_directory_to_run_in() const { return mDir; }
+        const std::vector<std::string>& get_command_to_run() const { return mCmdArgs; }
+        const std::map<std::string, std::string>& get_env_vars() const { return mEnvVars; }
+
+        void add_env_var(const std::string& key, const std::string& value) { mEnvVars[key] = value; }
+        
+        bool empty() const { return mCmdArgs.empty(); }
+
+    private:
+        std::string mDir;
+        std::vector<std::string> mCmdArgs;
+        std::map<std::string, std::string> mEnvVars;
+};
+
+
+} // namespace dropshell
+
+
+
+#endif

src/utils/hash.cpp

@@ -1,11 +1,15 @@
-#include "hash.hpp"
+#include "utils/hash.hpp"
+
+#define XXH_INLINE_ALL
+#include "contrib/xxhash.hpp"
+
 #include <fstream>
 #include <filesystem>
 #include <iostream>
 
 namespace dropshell {
 
-XXH64_hash_t hash_file(const std::string &path) {
+uint64_t hash_file(const std::string &path) {
     // Create hash state
     XXH64_state_t* const state = XXH64_createState();
     if (state == nullptr) {
@@ -14,11 +18,8 @@ XXH64_hash_t hash_file(const std::string &path) {
     }
 
     // Initialize state with seed 0
-    if (XXH64_reset(state, 0) == XXH_ERROR) {
+    XXH64_hash_t const seed = 0;   /* or any other value */
-        std::cerr << "Failed to reset hash state" << std::endl;
+    if (XXH64_reset(state, seed) == XXH_ERROR) return 0;
-        XXH64_freeState(state);
-        return 0;
-    }
 
     // Open file
     std::ifstream file(path, std::ios::binary);
@@ -54,7 +55,7 @@ XXH64_hash_t hash_file(const std::string &path) {
     return hash;
 }
 
-XXH64_hash_t hash_directory_recursive(const std::string &path) {
+uint64_t hash_directory_recursive(const std::string &path) {
     // Create hash state
     XXH64_state_t* const state = XXH64_createState();
     if (state == nullptr) {
@@ -63,7 +64,8 @@ XXH64_hash_t hash_directory_recursive(const std::string &path) {
     }
 
     // Initialize state with seed 0
-    if (XXH64_reset(state, 0) == XXH_ERROR) {
+    XXH64_hash_t const seed = 0;   /* or any other value */
+    if (XXH64_reset(state, seed) == XXH_ERROR) {
         std::cerr << "Failed to reset hash state" << std::endl;
         XXH64_freeState(state);
         return 0;
@@ -75,13 +77,7 @@ XXH64_hash_t hash_directory_recursive(const std::string &path) {
             if (entry.is_regular_file()) {
                 // Get file hash
                 XXH64_hash_t file_hash = hash_file(entry.path().string());
-                
+                XXH64_update(state, &file_hash, sizeof(file_hash));
-                // Update directory hash with file hash
-                if (XXH64_update(state, &file_hash, sizeof(file_hash)) == XXH_ERROR) {
-                    std::cerr << "Failed to update hash" << std::endl;
-                    XXH64_freeState(state);
-                    return 0;
-                }
             }
         }
     } catch (const std::filesystem::filesystem_error& e) {
@@ -96,11 +92,27 @@ XXH64_hash_t hash_directory_recursive(const std::string &path) {
     return hash;
 }
 
+uint64_t hash_path(const std::string &path) {
+    if (!std::filesystem::exists(path)) {
+        std::cerr << "Path does not exist: " << path << std::endl;
+        return 0;
+    }
+
+    if (std::filesystem::is_directory(path)) {
+        return hash_directory_recursive(path);
+    } else if (std::filesystem::is_regular_file(path)) {
+        return hash_file(path);
+    } else {
+        std::cerr << "Path is neither a file nor a directory: " << path << std::endl;
+        return 0;
+    }
+}
+
 void hash_demo(const std::string & path)
 {
-    std::cout << "Hashing directory: " << path << std::endl;
+    std::cout << "Hashing path: " << path << std::endl;
     auto start = std::chrono::high_resolution_clock::now();
-    XXH64_hash_t hash = hash_directory_recursive(path);
+    XXH64_hash_t hash = hash_path(path);
     auto end = std::chrono::high_resolution_clock::now();
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
     std::cout << "Hash: " << hash << " (took " << duration.count() << "ms)" << std::endl;
@@ -111,7 +123,7 @@ int hash_demo_raw(const std::string & path)
     if (!std::filesystem::exists(path)) {
         std::cout << 0 <<std::endl; return 1;
     }
-    XXH64_hash_t hash = hash_directory_recursive(path);
+    XXH64_hash_t hash = hash_path(path);
     std::cout << hash << std::endl;
     return 0;
 }

src/utils/hash.hpp

@@ -1,14 +1,16 @@
 #ifndef HASH_HPP
 #define HASH_HPP
 
-#include <xxhash.h>
 #include <string>
+#include <cstdint>
 
 namespace dropshell {
 
-    XXH64_hash_t hash_file(const std::string &path);
+    uint64_t hash_file(const std::string &path);
 
-    XXH64_hash_t hash_directory_recursive(const std::string &path);
+    uint64_t hash_directory_recursive(const std::string &path);
+
+    uint64_t hash_path(const std::string &path);
 
     void hash_demo(const std::string & path);
 

src/utils/json_fwd.hpp

@@ -0,0 +1,187 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
+#define INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+#include <cstdint> // int64_t, uint64_t
+#include <map> // map
+#include <memory> // allocator
+#include <string> // string
+#include <vector> // vector
+
+// #include <nlohmann/detail/abi_macros.hpp>
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+
+
+// This file contains all macro definitions affecting or depending on the ABI
+
+#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK
+    #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH)
+        #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 || NLOHMANN_JSON_VERSION_PATCH != 0
+            #warning "Already included a different version of the library!"
+        #endif
+    #endif
+#endif
+
+#define NLOHMANN_JSON_VERSION_MAJOR 3   // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_MINOR 12  // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_PATCH 0   // NOLINT(modernize-macro-to-enum)
+
+#ifndef JSON_DIAGNOSTICS
+    #define JSON_DIAGNOSTICS 0
+#endif
+
+#ifndef JSON_DIAGNOSTIC_POSITIONS
+    #define JSON_DIAGNOSTIC_POSITIONS 0
+#endif
+
+#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+    #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0
+#endif
+
+#if JSON_DIAGNOSTICS
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag
+#else
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS
+#endif
+
+#if JSON_DIAGNOSTIC_POSITIONS
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS _dp
+#else
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS
+#endif
+
+#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+    #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp
+#else
+    #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION
+    #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0
+#endif
+
+// Construct the namespace ABI tags component
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) json_abi ## a ## b ## c
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c) \
+    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c)
+
+#define NLOHMANN_JSON_ABI_TAGS                                       \
+    NLOHMANN_JSON_ABI_TAGS_CONCAT(                                   \
+            NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS,                       \
+            NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \
+            NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS)
+
+// Construct the namespace version component
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \
+    _v ## major ## _ ## minor ## _ ## patch
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \
+    NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch)
+
+#if NLOHMANN_JSON_NAMESPACE_NO_VERSION
+#define NLOHMANN_JSON_NAMESPACE_VERSION
+#else
+#define NLOHMANN_JSON_NAMESPACE_VERSION                                 \
+    NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \
+                                           NLOHMANN_JSON_VERSION_MINOR, \
+                                           NLOHMANN_JSON_VERSION_PATCH)
+#endif
+
+// Combine namespace components
+#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b
+#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \
+    NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b)
+
+#ifndef NLOHMANN_JSON_NAMESPACE
+#define NLOHMANN_JSON_NAMESPACE               \
+    nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \
+            NLOHMANN_JSON_ABI_TAGS,           \
+            NLOHMANN_JSON_NAMESPACE_VERSION)
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN
+#define NLOHMANN_JSON_NAMESPACE_BEGIN                \
+    namespace nlohmann                               \
+    {                                                \
+    inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \
+                NLOHMANN_JSON_ABI_TAGS,              \
+                NLOHMANN_JSON_NAMESPACE_VERSION)     \
+    {
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_END
+#define NLOHMANN_JSON_NAMESPACE_END                                     \
+    }  /* namespace (inline namespace) NOLINT(readability/namespace) */ \
+    }  // namespace nlohmann
+#endif
+
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+/// a class to store JSON values
+/// @sa https://json.nlohmann.me/api/basic_json/
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer,
+         class BinaryType = std::vector<std::uint8_t>, // cppcheck-suppress syntaxError
+         class CustomBaseClass = void>
+class basic_json;
+
+/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document
+/// @sa https://json.nlohmann.me/api/json_pointer/
+template<typename RefStringType>
+class json_pointer;
+
+/*!
+@brief default specialization
+@sa https://json.nlohmann.me/api/json/
+*/
+using json = basic_json<>;
+
+/// @brief a minimal map-like container that preserves insertion order
+/// @sa https://json.nlohmann.me/api/ordered_map/
+template<class Key, class T, class IgnoredLess, class Allocator>
+struct ordered_map;
+
+/// @brief specialization that maintains the insertion order of object keys
+/// @sa https://json.nlohmann.me/api/ordered_json/
+using ordered_json = basic_json<nlohmann::ordered_map>;
+
+NLOHMANN_JSON_NAMESPACE_END
+
+#endif  // INCLUDE_NLOHMANN_JSON_FWD_HPP_

src/utils/readmes.cpp

@@ -1,36 +0,0 @@
-#include "readmes.hpp"
-
-#include <fstream>
-#include <filesystem>
-void dropshell::create_readme_local_config_dir(const std::string &readme_path)
-{
-    if (std::filesystem::exists(readme_path)) 
-       return; // already exists
-       
-    std::ofstream readme_file(readme_path);
-    // use heredoc to write the readme
-    readme_file << R"(
-
-use this directory to store your local config files for your servers and services.
-  dropshell create-server <server_name>
-  dropshell create-template <template_name>
-  dropshell create-service <server_name> <template_name> <service_name>
-
-config_path
-    |-- servers
-    |   |-- server_name
-    |       |-- server.env
-    |       |-- services
-    |           |-- service_name
-    |               |-- service.env
-    |               |-- (other config files for specific server&service)
-    |-- templates
-    |   |-- template_name
-    |       |-- (script files)
-    |       |-- example
-    |           |-- service.env
-    |           |-- (other service config files)
-
-    )" << std::endl;
-    readme_file.close();
-}

src/utils/readmes.hpp

@@ -1,12 +0,0 @@
-#ifndef READMES_HPP
-#define READMES_HPP
-
-#include <string>
-
-namespace dropshell {
-
-    void create_readme_local_config_dir(const std::string &readme_path);
-
-} // namespace dropshell
-
-#endif

src/utils/utils.cpp

@@ -5,6 +5,9 @@
 #include <vector>
 #include <algorithm>
 #include <filesystem>
+#include <regex>
+#include <random>
+
 namespace dropshell {
 
 void maketitle(const std::string& title) {
@@ -72,6 +75,16 @@ std::string quote(std::string str)
     return "\""+str+"\"";
 }
 
+std::string halfquote(std::string str)
+{
+    return "'" + str + "'";
+}
+
+std::string escapequotes(std::string str)
+{
+    return std::regex_replace(str, std::regex("\""), "\\\"");
+}
+
 std::string multi2string(std::vector<std::string> values)
 {
     std::string result;
@@ -263,4 +276,44 @@ std::vector<std::string> split(const std::string& str, const std::string& delimi
     return tokens;
 }
 
+
+std::string replace_with_environment_variables_like_bash(std::string str) {
+    // Combined regex pattern for both ${var} and $var formats
+    std::regex var_pattern("\\$(?:\\{([^}]+)\\}|([a-zA-Z0-9_]+))");
+    std::string result = str;
+    std::smatch match;
+    
+    while (std::regex_search(result, match, var_pattern)) {
+        // match[1] will contain capture from ${var} format
+        // match[2] will contain capture from $var format
+        std::string var_name = match[1].matched ? match[1].str() : match[2].str();
+        
+        // Get value from system environment variables
+        const char* env_value = std::getenv(var_name.c_str());
+        std::string value = env_value ? env_value : "";
+        
+        result = result.replace(match.position(), match.length(), value);
+    }
+
+    // dequote the result
+    return result;
+}
+
+std::string random_alphanumeric_string(int length)
+{
+    static std::mt19937 generator(std::random_device{}());
+    static const std::string chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+    std::uniform_int_distribution<> distribution(0, chars.size() - 1);
+    std::string random_string;
+    for (int i = 0; i < length; ++i) {
+        random_string += chars[distribution(generator)];
+    }
+    
+    return random_string;
+}
+
+std::string requote(std::string str) {
+    return quote(trim(dequote(trim(str))));
+}
+
 } // namespace dropshell 

src/utils/utils.hpp

@@ -19,6 +19,10 @@ bool replace_line_in_file(const std::string& file_path, const std::string& searc
 std::string trim(std::string str);
 std::string dequote(std::string str);
 std::string quote(std::string str);
+std::string halfquote(std::string str);
+std::string requote(std::string str);
+std::string escapequotes(std::string str);
+
 std::string multi2string(std::vector<std::string> values);
 std::vector<std::string> string2multi(std::string values);
 std::vector<std::string> split(const std::string& str, const std::string& delimiter);
@@ -33,4 +37,8 @@ void ensure_directories_exist(std::vector<std::string> directories);
 std::vector<int> search(const std::string &pat, const std::string &txt);
 int count_substring(const std::string &substring, const std::string &text);
 
+std::string replace_with_environment_variables_like_bash(std::string str);
+
+std::string random_alphanumeric_string(int length);
+
 } // namespace dropshell 

templates/caddy/README.txt

@@ -0,0 +1,3 @@
+Caddy!
+
+Edit the static site, and the Caddyfile.

templates/caddy/_default.env

@@ -0,0 +1,8 @@
+# Service settings specific to this server 
+
+# Image settings
+IMAGE_REGISTRY="docker.io"
+IMAGE_REPO="caddy"
+
+DATA_VOLUME=caddy_data
+CONFIG_VOLUME=caddy_config

templates/caddy/backup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+_stop_container "$CONTAINER_NAME"
+
+autobackup "$1" "$2" "volume=$DATA_VOLUME" "volume=$CONFIG_VOLUME" || _die "Failed to create backup"
+
+_start_container "$CONTAINER_NAME"
+
+echo "Backup created successfully: $BACKUP_FILE"

templates/caddy/config/.template_info.env

@@ -0,0 +1,2 @@
+# Template to use - always required!
+TEMPLATE=caddy

templates/caddy/config/Caddyfile

@@ -0,0 +1,6 @@
+# See https://caddyserver.com/docs/caddyfile
+
+localhost {
+root * /srv
+file_server
+}

templates/caddy/config/service.env

@@ -0,0 +1,10 @@
+# Service settings specific to this server 
+# (can also override anything in the _default.env file in the template to make it specific to this server)
+CONTAINER_NAME=caddy
+IMAGE_TAG="latest"
+
+# Scripts will have these environment variables set, plus those in _default.env, plus:
+# SERVER, SERVICE, CONFIG_PATH
+# CONFIG_PATH points to this directory!
+
+

templates/caddy/config/static/index.html

@@ -0,0 +1,9 @@
+<html>
+    <head>
+        <title>Static Site</title>
+    </head>
+    <body>
+        <h1>Static Site</h1>
+        <p>This is a static site.</p>
+    </body>
+</html>

templates/caddy/install.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" "DATA_VOLUME" "CONFIG_VOLUME" "CONFIG_PATH"
+
+autocreate "volume=$DATA_VOLUME" "volume=$CONFIG_VOLUME" || _die "Failed to autocreate volumes $DATA_VOLUME and $CONFIG_VOLUME"
+
+_check_docker_installed || _die "Docker test failed, aborting installation..."
+
+docker pull "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || _die "Failed to pull image $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG"
+
+[ -f "${CONFIG_PATH}/Caddyfile" ] || _die "Caddyfile not found in ${CONFIG_PATH}!"
+
+bash ./stop.sh || _die "Failed to stop container ${CONTAINER_NAME}"
+_remove_container $CONTAINER_NAME || _die "Failed to remove container ${CONTAINER_NAME}"
+bash ./start.sh || _die "Failed to start container ${CONTAINER_NAME}"
+
+echo "Installation of ${CONTAINER_NAME} complete"

templates/caddy/logs.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# Main script.
+echo "Container ${CONTAINER_NAME} logs:"
+_grey_start
+docker logs "${CONTAINER_NAME}"
+_grey_end

templates/caddy/nuke.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# NUKE SCRIPT
+# This is run after the uninstall.sh script to delete all data.
+# dropshell handles the configuration files, so we just need to remove
+# any docker volumes and any custom local data folders.
+
+
+autonuke "volume=$DATA_VOLUME" "volume=$CONFIG_VOLUME" || _die "Failed to nuke"
+
+echo "Nuking of ${CONTAINER_NAME} complete."

templates/caddy/ports.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+
+
+echo 80
+echo 443

templates/caddy/restore.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# RESTORE SCRIPT
+
+
+# uninstall container before restore
+bash ./uninstall.sh || _die "Failed to uninstall service before restore"
+
+# restore data from backup file
+autorestore "$1" "$2" "volume=$DATA_VOLUME" "volume=$CONFIG_VOLUME" || _die "Failed to restore data from backup file"
+
+# reinstall service
+bash ./install.sh || _die "Failed to reinstall service after restore"
+
+echo "Restore complete! Service is running again."

templates/caddy/start.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# START SCRIPT
+# The start script is required for all templates.
+# It is used to start the service on the server.
+
+
+
+DOCKER_RUN_CMD="docker run -d \
+                --restart unless-stopped \
+                --name ${CONTAINER_NAME} \
+                -p 80:80 \
+                -p 443:443 \
+                -p 443:443/udp \
+                -v ${CONFIG_PATH}/Caddyfile:/etc/caddy/Caddyfile \
+                -v ${DATA_VOLUME}:/data \
+                -v ${CONFIG_VOLUME}:/config \
+                -v ${CONFIG_PATH}/static:/srv \
+                ${IMAGE_REGISTRY}/${IMAGE_REPO}:${IMAGE_TAG}"
+
+
+if ! _create_and_start_container "$DOCKER_RUN_CMD" "$CONTAINER_NAME"; then
+    _die "Failed to start container ${CONTAINER_NAME}"
+fi
+
+# Check if the container is running
+if ! _is_container_running "$CONTAINER_NAME"; then
+    _die "Container ${CONTAINER_NAME} is not running"
+fi
+
+echo "Container ${CONTAINER_NAME} started"

templates/caddy/status.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# STATUS SCRIPT
+# The status script is OPTIONAL.
+# It is used to return the status of the service (0 is healthy, 1 is unhealthy).
+
+# This is an example of a status script that checks if the service is running.
+
+# check if the service is running
+_is_container_running $CONTAINER_NAME || _die "Service is not running - did not find container $CONTAINER_NAME."
+
+echo "Service is healthy"
+exit 0

templates/caddy/stop.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# STOP SCRIPT
+# The stop script is required for all templates.
+# It is used to stop the service on the server.
+
+
+_stop_container $CONTAINER_NAME || _die "Failed to stop container ${CONTAINER_NAME}"
+
+echo "Container ${CONTAINER_NAME} stopped"

templates/caddy/uninstall.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars
+
+# UNINSTALL SCRIPT
+# The uninstall script is required for all templates.
+# It is used to uninstall the service from the server.
+
+
+_remove_container $CONTAINER_NAME || _die "Failed to remove container ${CONTAINER_NAME}"
+_is_container_running && _die "Couldn't stop existing container"
+_is_container_exists && _die "Couldn't remove existing container"
+
+# remove the image
+docker rmi "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || echo "Failed to remove image $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG"
+
+echo "Uninstallation of ${CONTAINER_NAME} complete."
+echo "Local data still in place."

templates/dropshell-agent/_allservicesstatus.sh

@@ -17,7 +17,7 @@ SCRIPT_DIR="$(dirname "$0")"
 # //               |-- service.env
 # //           |-- template
 # //               |-- (script files)
-# //               |-- example
+# //               |-- config
 # //                   |-- service.env
 # //                   |-- (other config files for specific server&service)
 

templates/dropshell-agent/_nuke_other.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/shared/_common.sh"
+
+
+A_SERVICE="$1"
+A_SERVICE_PATH="$2"
+
+
+# 1. Check if service directory exists on server
+[ -d "$A_SERVICE_PATH" ] || _die "Service is not installed: $A_SERVICE"
+
+# uninstall the service
+if [ -f "$A_SERVICE_PATH/uninstall.sh" ]; then
+    $A_SERVICE_PATH/uninstall.sh
+fi
+
+# nuke the service
+if [ -f "$A_SERVICE_PATH/nuke.sh" ]; then
+    $A_SERVICE_PATH/nuke.sh
+fi
+
+# remove the service directory
+rm -rf "$A_SERVICE_PATH"

templates/dropshell-agent/install.sh

@@ -5,4 +5,25 @@
 # It is used to install the service on the server.
 # It is called with the path to the server specific env file as an argument.
 
-echo "Installation of dropshell-agent complete"
+
+check_prerequisites() {
+    # prerequisites:
+    # - bash
+    # - curl
+    # - wget
+    # - docker
+
+    PREREQUISITES=("bash" "curl" "wget" "jq" "docker")
+
+    # check if all prerequisites are installed
+    for prerequisite in "${PREREQUISITES[@]}"; do
+        if ! command -v "${prerequisite}" &> /dev/null; then
+            echo "Prerequisite: ${prerequisite} is not installed."
+            exit 1
+        fi
+    done
+}
+
+check_prerequisites
+exit 0
+

templates/dropshell-agent/shared/_autocommands.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+
+# This script contains the common code for the autocommands.
+_check_required_env_vars "BACKUP_FILE" "TEMP_DIR"
+
+MYID=$(id -u)
+MYGRP=$(id -g)
+
+BACKUP_TEMP_PATH="$TEMP_DIR/backup"
+
+_autocommandrun_volume() {
+    local command="$1"
+    local volume_name="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            echo "Creating volume ${volume_name}"
+            docker volume create ${volume_name}
+            ;;
+        nuke)
+            echo "Nuking volume ${volume_name}"
+            docker volume rm ${volume_name}
+            ;;
+        backup)
+            echo "Backing up volume ${volume_name}"
+            docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /volume . && chown -R $MYID:$MYGRP /backup"
+            ;;
+        restore)
+            echo "Restoring volume ${volume_name}"
+            docker volume rm ${volume_name}
+            docker volume create ${volume_name}
+            docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -xzvf /backup/backup.tgz -C /volume --strip-components=1"
+            ;;
+    esac
+}   
+
+_autocommandrun_path() {
+    local command="$1"
+    local path="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            echo "Creating path ${path}"
+            mkdir -p ${path}
+            ;;
+        nuke)
+            echo "Nuking path ${path}"
+            PATHPARENT=$(dirname ${path})
+            PATHCHILD=$(basename ${path})
+            if [ -d "${PATHPARENT}/${PATHCHILD}" ]; then
+                docker run --rm -v ${PATHPARENT}:/volume debian bash -c "rm -rf /volume/${PATHCHILD}" || echo "Failed to nuke path ${path}"
+            else
+                echo "Path ${path} does not exist - nothing to nuke"
+            fi
+            ;;
+        backup)
+            echo "Backing up path ${path}"
+            if [ -d "${path}" ]; then
+                docker run --rm -v ${path}:/path -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /path . && chown -R $MYID:$MYGRP /backup"
+            else
+                echo "Path ${path} does not exist - nothing to backup"
+            fi
+            ;;
+        restore)
+            echo "Restoring path ${path}"
+            tar -xzvf ${backup_folder}/backup.tgz -C ${path} --strip-components=1
+            ;;
+    esac
+}
+
+_autocommandrun_file() {
+    local command="$1"
+    local filepath="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            ;;
+        nuke)
+            rm -f ${filepath}
+            ;;
+        backup)
+            echo "Backing up file ${filepath}"
+            FILEPARENT=$(dirname ${filepath})
+            FILENAME=$(basename ${filepath})
+            if [ -f "${FILEPARENT}/${FILENAME}" ]; then
+                docker run --rm-v ${FILEPARENT}:/volume -v ${backup_folder}:/backup debian bash -c "cp /volume/${FILENAME} /backup/${FILENAME} && chown -R $MYID:$MYGRP /backup"
+            else
+                echo "File ${filepath} does not exist - nothing to backup"
+            fi
+            ;;
+        restore)
+            echo "Restoring file ${filepath}"
+            local FILENAME=$(basename ${filepath})
+            cp ${backup_folder}/${FILENAME} ${filepath}
+            ;;
+    esac
+}
+
+_autocommandparse() {
+    # first argument is the command
+    # if the command is backup or restore, then the last two arguments are the backup file and the temporary path
+    # all other arguments are of form:
+    # key=value
+    # where key can be one of volume, path or file.
+    # value is the path or volume name.
+
+    # we iterate over the key=value arguments, and for each we call:
+    #    autorun <command> <backupfile> <key> <value>
+
+    local command="$1"
+    shift
+    echo "autocommandparse: command=$command"
+
+    # Extract the backup file and temp path (last two arguments)
+    local args=("$@")
+    local arg_count=${#args[@]}
+    
+    # Process all key=value pairs
+    for ((i=0; i<$arg_count; i++)); do
+        local pair="${args[$i]}"
+        
+        # Skip if not in key=value format
+        if [[ "$pair" != *"="* ]]; then
+            continue
+        fi
+        
+        local key="${pair%%=*}"
+        local value="${pair#*=}"
+
+        # create backup folder unique to key/value.
+        local bfolder=$(echo "${key}_${value}" | tr -cd '[:alnum:]_-')
+        local targetpath="${BACKUP_TEMP_PATH}/${bfolder}"
+        mkdir -p ${targetpath}
+
+        # Key must be one of volume, path or file
+        case "$key" in
+            volume)
+                _autocommandrun_volume "$command" "$value" "$targetpath"
+                ;;
+            path)
+                _autocommandrun_path "$command" "$value" "$targetpath"
+                ;;
+            file)
+                _autocommandrun_file "$command" "$value" "$targetpath"
+                ;;
+            *)
+                _die "Unknown key $key passed to auto${command}. We only support volume, path and file."
+                ;;
+        esac
+    done
+}
+
+
+autocreate() {
+    _autocommandparse create "$@"
+}
+
+
+autonuke() {
+    _autocommandparse nuke "$@"
+}
+
+
+autobackup() {
+    mkdir -p "$BACKUP_TEMP_PATH"
+    echo "_autocommandparse [backup] [$@]"
+    _autocommandparse backup "$@"
+
+    tar zcvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" .
+}
+
+
+autorestore() {
+    echo "_autocommandparse [restore] [$@]"
+
+    mkdir -p "$BACKUP_TEMP_PATH"
+    tar zxvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" --strip-components=1
+
+    _autocommandparse restore "$@"
+}
+
+

templates/dropshell-agent/shared/_common.sh

@@ -0,0 +1,183 @@
+# COMMON FUNCTIONS
+# JDE
+# 2025-05-03
+
+# This file is available TO ***ALL*** templates, as ${AGENT_PATH}/_common.sh
+
+# ----------------------------------------------------------------------------------------------------------
+
+# summary of functions:
+#   _die "message"                         : Prints an error message in red and exits with status code 1.
+#   _grey_start                            : Switches terminal output color to grey.
+#   _grey_end                              : Resets terminal output color from grey.
+#   _create_and_start_container "<run_cmd>" <container_name> : Creates/starts a container, verifying it runs.
+#   _create_folder <folder_path>           : Creates a directory if it doesn't exist (chmod 777).
+#   _check_docker_installed               : Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
+#   _is_container_exists <container_name> : Checks if a container (any state) exists. Returns 1 if not found.
+#   _is_container_running <container_name>: Checks if a container is currently running. Returns 1 if not running.
+#   _get_container_id <container_name>    : Prints the ID of the named container.
+#   _get_container_status <container_name>: Prints the status string of the named container.
+#   _start_container <container_name>     : Starts an existing, stopped container.
+#   _stop_container <container_name>      : Stops a running container.
+#   _remove_container <container_name>    : Stops (if needed) and removes a container.
+#   _get_container_logs <container_name>  : Prints the logs for a container.
+#   _check_required_env_vars "VAR1" ...    : Checks if listed environment variables are set; calls _die() if any are missing.
+#   _root_remove_tree <path>              : Removes a path using a root Docker container (for permissions).
+
+# ----------------------------------------------------------------------------------------------------------
+
+# Prints an error message in red and exits with status code 1.
+_die() {
+    echo -e "\033[91mError: $1\033[0m"
+    exit 1
+}
+
+# Switches terminal output color to grey.
+_grey_start() {
+    echo -e -n "\033[90m"
+}
+
+# Resets terminal output color from grey.
+_grey_end() {
+    echo -e -n "\033[0m"
+}
+
+# Creates/starts a container, verifying it runs.
+_create_and_start_container() {
+    if [ -z "$1" ] || [ -z "$2" ]; then
+        _die "Template error: create_and_start_container <run_cmd> <container_name>"
+    fi
+
+    local run_cmd="$1"
+    local container_name="$2"
+
+    if _is_container_exists $container_name; then
+        _is_container_running $container_name && return 0
+        _start_container $container_name
+    else
+        _grey_start
+        $run_cmd
+        _grey_end
+    fi
+
+    if ! _is_container_running $container_name; then
+        _die "Container ${container_name} failed to start"
+    fi
+
+    ID=$(_get_container_id $container_name)
+    echo "Container ${container_name} is running with ID ${ID}"
+}
+
+# Creates a directory if it doesn't exist (chmod 777).
+_create_folder() {
+    local folder="$1"
+    if [ -d "$folder" ]; then
+        return 0
+    fi
+    if ! mkdir -p "$folder"; then
+        _die "Failed to create folder: $folder"
+    fi
+    chmod 777 "$folder"
+    echo "Folder created: $folder"
+}
+
+# Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
+_check_docker_installed() {
+    if ! command -v docker &> /dev/null; then
+        echo "Docker is not installed"
+        return 1
+    fi
+
+    # check if docker daemon is running
+    if ! docker info &> /dev/null; then
+        echo "Docker daemon is not running"
+        return 1
+    fi
+
+    # check if user has permission to run docker
+    if ! docker run --rm hello-world &> /dev/null; then
+        echo "User does not have permission to run docker"
+        return 1
+    fi
+
+    return 0
+}
+
+# Checks if a container (any state) exists. Returns 1 if not found.
+_is_container_exists() {
+    if ! docker ps -a --format "{{.Names}}" | grep -q "^$1$"; then
+        return 1
+    fi
+    return 0
+}
+
+# Checks if a container is currently running. Returns 1 if not running.
+_is_container_running() {
+    if ! docker ps --format "{{.Names}}" | grep -q "^$1$"; then
+        return 1
+    fi
+    return 0
+}
+
+# Prints the ID of the named container.
+_get_container_id() {
+    docker ps --format "{{.ID}}" --filter "name=$1"
+}
+
+# Prints the status string of the named container.
+_get_container_status() {
+    docker ps --format "{{.Status}}" --filter "name=$1"
+}
+
+# Starts an existing, stopped container.
+_start_container() {
+    _is_container_exists $1 || return 1
+    _is_container_running $1 && return 0
+    docker start $1
+}
+
+# Stops a running container.
+_stop_container() {
+    _is_container_running $1 || return 0;
+    docker stop $1
+}   
+
+# Stops (if needed) and removes a container.
+_remove_container() {
+    _stop_container $1
+    _is_container_exists $1 || return 0;
+    docker rm $1
+}
+
+# Prints the logs for a container.
+_get_container_logs() {
+    if ! _is_container_exists $1; then
+        echo "Container $1 does not exist"
+        return 1
+    fi
+
+    docker logs $1
+}
+
+# Checks if listed environment variables are set; calls _die() if any are missing.
+_check_required_env_vars() {
+    local required_vars=("$@")
+    for var in "${required_vars[@]}"; do
+        if [ -z "${!var}" ]; then
+            _die "Required environment variable $var is not set"
+        fi
+    done
+}
+
+# Removes a path using a root Docker container (for permissions).
+_root_remove_tree() {
+    local to_remove="$1"
+    parent=$(dirname "$to_remove")
+    abs_parent=$(realpath "$parent")
+    child=$(basename "$to_remove")
+    docker run --rm -v "$abs_parent":/data alpine rm -rf "/data/$child"
+}
+
+
+# Load autocommands
+source "${AGENT_PATH}/_autocommands.sh"

templates/dropshell-agent/shared/_run.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+directory_to_run_in=${1:-}
+command_to_run_base64=${2:-}
+
+if [ -z "$directory_to_run_in" ]; then
+    echo "Usage: $0 <directory_to_run_in> <command_to_run>"
+    exit 1
+fi
+
+if [ -z "$command_to_run_base64" ]; then
+    echo "Usage: $0 <directory_to_run_in> <command_to_run>"
+    exit 1
+fi
+
+command_to_run=$(echo "$command_to_run_base64" | base64 -d)
+
+if [ -z "$command_to_run" ]; then
+    echo "Usage: $0 <directory_to_run_in> <command_to_run>"
+    exit 1
+fi
+
+
+if [ -n "$directory_to_run_in" ]; then
+    cd "$directory_to_run_in"
+fi
+
+eval "$command_to_run"
+
+
+
+

templates/dropshell-agent/uninstall.sh

@@ -1,7 +1 @@
 #!/bin/bash
-
-# UNINSTALL SCRIPT
-# The uninstall script is required for all templates.
-# It is used to uninstall the service from the server.
-# It is called with the path to the server specific env file as an argument.
-

templates/example-nginx/README.txt

@@ -1,33 +1 @@
-DropShell Template Example - Nginx simple webserver
+Caddy
-
-A simple service example, creating a single docker container running nginx, with contant on the host
-in the configurable LOCAL_DATA_FOLDER directory.
-
-Shell scripts defined in this folder are run as DropShell commands on the remote server (not locally!).
-
-When they are run, the following environment variables will be set:
-- SERVER (server name), SERVICE (service name) and CONFIG_PATH (path to the user's service configuration folder)
-- everything in _default.env
-- everything in the server's particular service.env file (defaults in example/service.env)
-
-The optional backup and restore scripts get a second argument, which is the backup file to create/restore
-(must be a single tgz file).
-
-Mandatory scripts are:
-- install.sh
-- uninstall.sh
-
-Optional standard scripts are:
-- start.sh
-- stop.sh
-- backup.sh
-- restore.sh
-- status.sh
-- ports.sh
-- logs.sh
-
-
-The example/ folder gets copied to the service's configuration, edited for the particular server settings,
-then copied onto the server. The location is server-specific, but can be accessed via CONFIG_PATH.
-
-You can use it to store things like an nginx config file (this example does not).

templates/example-nginx/_common.sh

@@ -1,155 +0,0 @@
-#!/bin/bash
-
-# COMMON FUNCTIONS
-# JDE
-# 2025-04-25
-
-# This file is not required if you write your own template.
-
-
-# Print error message and exit with code 1
-# Usage: die "error message"
-die() {
-    echo -e "\033[91mError: $1\033[0m"
-    exit 1
-}
-
-grey_start() {
-    echo -e -n "\033[90m"
-}
-
-grey_end() {
-    echo -e -n "\033[0m"
-}
-
-create_and_start_container() {
-    if [ -z "$1" ] || [ -z "$2" ]; then
-        die "Template error: create_and_start_container <run_cmd> <container_name>"
-    fi
-
-    local run_cmd="$1"
-    local container_name="$2"
-
-    if _is_container_exists $container_name; then
-        _is_container_running $container_name && return 0
-        _start_container $container_name
-    else
-        grey_start
-        $run_cmd
-        grey_end
-    fi
-
-    if ! _is_container_running $container_name; then
-        die "Container ${container_name} failed to start"
-    fi
-
-    ID=$(_get_container_id $container_name)
-    echo "Container ${container_name} is running with ID ${ID}"
-}
-
-function create_folder() {
-    local folder="$1"
-    if [ -d "$folder" ]; then
-        return 0
-    fi
-    if ! mkdir -p "$folder"; then
-        die "Failed to create folder: $folder"
-    fi
-    chmod 777 "$folder"
-    echo "Folder created: $folder"
-}
-
-# Check if docker is installed
-_check_docker_installed() {
-    if ! command -v docker &> /dev/null; then
-        echo "Docker is not installed"
-        return 1
-    fi
-
-    # check if docker daemon is running
-    if ! docker info &> /dev/null; then
-        echo "Docker daemon is not running"
-        return 1
-    fi
-
-    # check if user has permission to run docker
-    if ! docker run --rm hello-world &> /dev/null; then
-        echo "User does not have permission to run docker"
-        return 1
-    fi
-
-    return 0
-}
-
-# Check if a container exists
-_is_container_exists() {
-    if ! docker ps -a --format "{{.Names}}" | grep -q "^$1$"; then
-        return 1
-    fi
-    return 0
-}
-
-# Check if a container is running
-_is_container_running() {
-    if ! docker ps --format "{{.Names}}" | grep -q "^$1$"; then
-        return 1
-    fi
-    return 0
-}
-
-# get contianer ID
-_get_container_id() {
-    docker ps --format "{{.ID}}" --filter "name=$1"
-}
-
-# get container status
-_get_container_status() {
-    docker ps --format "{{.Status}}" --filter "name=$1"
-}
-
-# start container that exists
-_start_container() {
-    _is_container_exists $1 || return 1
-    _is_container_running $1 && return 0
-    docker start $1
-}
-
-# stop container that exists
-_stop_container() {
-    _is_container_running $1 || return 0;
-    docker stop $1
-}   
-
-# remove container that exists
-_remove_container() {
-    _stop_container $1
-    _is_container_exists $1 || return 0;
-    docker rm $1
-}
-
-# get container logs
-_get_container_logs() {
-    if ! _is_container_exists $1; then
-        echo "Container $1 does not exist"
-        return 1
-    fi
-
-    docker logs $1
-}
-
-check_required_env_vars() {
-    local required_vars=("$@")
-    for var in "${required_vars[@]}"; do
-        if [ -z "${!var}" ]; then
-            die "Required environment variable $var is not set in your service.env file"
-        fi
-    done
-}
-
-function _root_remove_tree() {
-    local to_remove="$1"
-    parent=$(dirname "$to_remove")
-    abs_parent=$(realpath "$parent")
-    child=$(basename "$to_remove")
-    docker run --rm -v "$abs_parent":/data alpine rm -rf "/data/$child"
-}

templates/example-nginx/backup.sh

@@ -1,36 +1,10 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "LOCAL_DATA_FOLDER" "BACKUP_FILE" "TEMP_DIR" "CONTAINER_NAME"
 
-# BACKUP SCRIPT
+# Nginx Example Backup Script
-# The backup script is OPTIONAL.
+# hot backup is fine for nginx website content.
-# It is used to backup the service on the server.
-# It is called with one argument: the path to the destination backup file.
-# If the backup file already exists, the script should exit with a message.
 
-source "$(dirname "$0")/_common.sh"
+autobackup "path=${LOCAL_DATA_FOLDER}" || _die "Failed to create backup"
-check_required_env_vars "CONTAINER_NAME" "LOCAL_DATA_FOLDER"
 
-# Get backup file path from first argument
+echo "Backup complete for ${CONTAINER_NAME}"
-BACKUP_FILE="$1"
-if [ -z "$BACKUP_FILE" ]; then
-    die "Backup file path not provided"
-fi
-
-# Check if backup file already exists
-if [ -f "$BACKUP_FILE" ]; then
-    die "Backup file $BACKUP_FILE already exists"
-fi
-
-# Stop container before backup
-_stop_container "$CONTAINER_NAME"
-
-Create backup of data folder
-echo "Creating backup of $LOCAL_DATA_FOLDER..."
-if ! tar zcvf "$BACKUP_FILE" -C "$LOCAL_DATA_FOLDER" .; then
-    _start_container "$CONTAINER_NAME"
-    die "Failed to create backup"
-fi
-
-# Start container after backup
-_start_container "$CONTAINER_NAME"
-
-echo "Backup created successfully: $BACKUP_FILE"

templates/example-nginx/config/.template_info.env

@@ -0,0 +1,2 @@
+# Template to use - always required!
+TEMPLATE=example-nginx

templates/example-nginx/config/service.env

@@ -1,11 +1,8 @@
-# Template to use - always required!
-TEMPLATE=example-nginx
-
 # Service settings specific to this server 
 # (can also override anything in the _default.env file in the template to make it specific to this server)
 HOST_PORT=60123
-LOCAL_DATA_FOLDER="${HOME}/.example"
+LOCAL_DATA_FOLDER="/home/dropshell/nginx-example-website"
-CONTAINER_NAME=example-nginx
+CONTAINER_NAME="example-nginx"
 IMAGE_TAG="latest"
 
 # Scripts will have these environment variables set, plus those in _default.env, plus:

templates/example-nginx/install.sh

@@ -1,41 +1,23 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "LOCAL_DATA_FOLDER" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" "CONTAINER_NAME"
 
-# INSTALL SCRIPT
+# Nginx Example Install Script
-# The install script is required for all templates.
-# It is used to install the service on the server.
-# It is called with the path to the server specific env file as an argument.
 
-source "$(dirname "$0")/_common.sh"
+# Ensure local data folder exists
+autocreate "path=${LOCAL_DATA_FOLDER}"
 
-# Required environment variables
+echo "Checking Docker installation..."
-check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" "LOCAL_DATA_FOLDER"
+_check_docker_installed || _die "Docker test failed, aborting installation..."
 
-# Create local data folder if it doesn't exist
+echo "Pulling image ${IMAGE_REGISTRY}/${IMAGE_REPO}:${IMAGE_TAG}..."
-if [ -d "${LOCAL_DATA_FOLDER}" ]; then
+docker pull "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || _die "Failed to pull image $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG"
-    echo "Local data folder ${LOCAL_DATA_FOLDER} exists, using existing data."
-else
-    echo "Local data folder ${LOCAL_DATA_FOLDER} does not exist, creating..."
-    mkdir -p "${LOCAL_DATA_FOLDER}"
-    cat <<EOF > "${LOCAL_DATA_FOLDER}/index.html"
-<html>
-<body>
-<h1>Hello, World!</h1>
-</body>
-</html>
-EOF
-fi
 
-# Test Docker 
+echo "Stopping and removing any existing container..."
-_check_docker_installed || die "Docker test failed, aborting installation..."
+bash ./stop.sh || _die "Failed to stop container ${CONTAINER_NAME}"
+_remove_container $CONTAINER_NAME || _die "Failed to remove container ${CONTAINER_NAME}"
 
+echo "Starting container..."
+bash ./start.sh || _die "Failed to start container ${CONTAINER_NAME}"
 
-# check can pull image on remote host and exit if fails
+echo "Installation complete for service ${CONTAINER_NAME}."
-docker pull "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || die "Failed to pull image $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG"
-
-# remove and restart, as the env may have changed.
-bash ./stop.sh || die "Failed to stop container ${CONTAINER_NAME}"
-_remove_container $CONTAINER_NAME || die "Failed to remove container ${CONTAINER_NAME}"
-bash ./start.sh || die "Failed to start container ${CONTAINER_NAME}"
-
-echo "Installation of ${CONTAINER_NAME} complete"
-echo "You can access the service at http://${SERVER}:${HOST_PORT}"

templates/example-nginx/logs.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME"
 
-# LOGS SCRIPT
+# Nginx Example Logs Script
-# The logs script is OPTIONAL.
-# It is used to return the logs of the service.
-# It is called with the path to the server specific env file as an argument.
 
-source "$(dirname "$0")/_common.sh"
+echo "Showing logs for ${CONTAINER_NAME}... (Press Ctrl+C to stop)"
-
+_grey_start
-# Required environment variables
+_get_container_logs $CONTAINER_NAME
-check_required_env_vars "CONTAINER_NAME"
+_grey_end
-
-echo "Container ${CONTAINER_NAME} logs:"
-grey_start
-docker logs "${CONTAINER_NAME}"
-grey_end

templates/example-nginx/nuke.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "LOCAL_DATA_FOLDER" "CONTAINER_NAME"
+
+# Nginx Example Nuke Script
+# Removes container and local data folder.
+
+# Call uninstall script first
+./uninstall.sh
+
+autonuke "path=${LOCAL_DATA_FOLDER}" || _die "Failed to nuke ${LOCAL_DATA_FOLDER}"
+
+echo "Nuke complete for service ${CONTAINER_NAME}."

templates/example-nginx/ports.sh

@@ -1,13 +1,7 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "HOST_PORT"
 
-# PORT SCRIPT
+# Nginx Example Ports Script
-# The port script is OPTIONAL.
-# It is used to return the ports used by the service.
-# It is called with the path to the server specific env file as an argument.
-
-source "$(dirname "$0")/_common.sh"
-
-# Required environment variables
-# check_required_env_vars "HOST_PORT" 
 
 echo $HOST_PORT

templates/example-nginx/restore.sh

@@ -1,41 +1,15 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "LOCAL_DATA_FOLDER" "BACKUP_FILE" "TEMP_DIR" "CONTAINER_NAME"
 
-# RESTORE SCRIPT
+# Nginx Example Restore Script
-# The restore script is OPTIONAL.
-# It is used to restore the service on the server from a backup file.
-# It is called with one argument: the path to the backup file.
 
-source "$(dirname "$0")/_common.sh"
+echo "Uninstalling service before restore..."
-check_required_env_vars "CONTAINER_NAME" "LOCAL_DATA_FOLDER"
+bash ./uninstall.sh || _die "Failed to uninstall service before restore"
 
-# Get backup file path from first argument
+autorestore "path=${LOCAL_DATA_FOLDER}" || _die "Failed to restore data folder from backup"
-BACKUP_FILE="$1"
-if [ -z "$BACKUP_FILE" ]; then
-    die "Backup file path not provided"
-fi
 
-# Check if backup file already exists
+echo "Restore complete. Reinstalling service..."
-if [ ! -f "$BACKUP_FILE" ]; then
+bash ./install.sh || _die "Failed to reinstall service after restore"
-    die "Backup file $BACKUP_FILE does not exist"
-fi
 
-# # Stop container before backup
+echo "Service ${CONTAINER_NAME} restored and reinstalled."
-bash ./uninstall.sh || die "Failed to uninstall service before restore"
-
-# Remove existing data folder
-echo "Deleting ALL data in $LOCAL_DATA_FOLDER."
-_root_remove_tree "$LOCAL_DATA_FOLDER"
-[ ! -d "$LOCAL_DATA_FOLDER" ] || die "Failed to delete $LOCAL_DATA_FOLDER"
-mkdir -p "$LOCAL_DATA_FOLDER"
-[ -d "$LOCAL_DATA_FOLDER" ] || die "Failed to create $LOCAL_DATA_FOLDER"
-
-# Restore data folder from backup
-#  --strip-components=1  removes the parent folder in the tgz from the restore paths.
-if ! tar xzvf "$BACKUP_FILE" -C "$LOCAL_DATA_FOLDER" --strip-components=1; then
-    die "Failed to restore data folder from backup"
-fi
-
-# reinstall service
-bash ./install.sh || die "Failed to reinstall service after restore"
-
-echo "Restore complete! Service is running again on port $HOST_PORT with restored website."

templates/example-nginx/start.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME" "LOCAL_DATA_FOLDER" "HOST_PORT" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG"
 
 # START SCRIPT
 # The start script is required for all templates.
 # It is used to start the service on the server.
 # It is called with the path to the server specific env file as an argument.
 
-source "$(dirname "$0")/_common.sh"
-check_required_env_vars "CONTAINER_NAME" "HOST_PORT" "LOCAL_DATA_FOLDER"
 
-[ -d "${LOCAL_DATA_FOLDER}" ] || die "Local data folder ${LOCAL_DATA_FOLDER} does not exist."
+[ -d "${LOCAL_DATA_FOLDER}" ] || _die "Local data folder ${LOCAL_DATA_FOLDER} does not exist."
 
 DOCKER_RUN_CMD="docker run -d \
                 --restart unless-stopped \
@@ -18,13 +18,13 @@ DOCKER_RUN_CMD="docker run -d \
                 ${IMAGE_REGISTRY}/${IMAGE_REPO}:${IMAGE_TAG}"
 
 
-if ! create_and_start_container "$DOCKER_RUN_CMD" "$CONTAINER_NAME"; then
+if ! _create_and_start_container "$DOCKER_RUN_CMD" "$CONTAINER_NAME"; then
-    die "Failed to start container ${CONTAINER_NAME}"
+    _die "Failed to start container ${CONTAINER_NAME}"
 fi
 
 # Check if the container is running
 if ! _is_container_running "$CONTAINER_NAME"; then
-    die "Container ${CONTAINER_NAME} is not running"
+    _die "Container ${CONTAINER_NAME} is not running"
 fi
 
 echo "Container ${CONTAINER_NAME} started"

templates/example-nginx/status.sh

@@ -1,21 +1,13 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME"
 
 # STATUS SCRIPT
 # The status script is OPTIONAL.
 # It is used to return the status of the service (0 is healthy, 1 is unhealthy).
-# It is called with the path to the server specific env file as an argument.
-
-
-# This is an example of a status script that checks if the service is running.
-source "$(dirname "$0")/_common.sh"
-check_required_env_vars "CONTAINER_NAME"
 
 # check if the service is running
-_is_container_running $CONTAINER_NAME || die "Service is not running - did not find container $CONTAINER_NAME."
+_is_container_running $CONTAINER_NAME || _die "Service is not running - did not find container $CONTAINER_NAME."
-
-# check if the service is healthy
-# curl -s -X GET http://localhost:${HOST_PORT}/health | grep -q "OK" \
-#     || die "Service is not healthy - did not get OK response from /health endpoint."
 
 echo "Service is healthy"
 exit 0

templates/example-nginx/stop.sh

@@ -1,13 +1,7 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME"
 
-# STOP SCRIPT
+_stop_container $CONTAINER_NAME || _die "Failed to stop container ${CONTAINER_NAME}"
-# The stop script is required for all templates.
-# It is used to stop the service on the server.
-# It is called with the path to the server specific env file as an argument.
-
-source "$(dirname "$0")/_common.sh"
-check_required_env_vars "CONTAINER_NAME"
-
-_stop_container $CONTAINER_NAME || die "Failed to stop container ${CONTAINER_NAME}"
 
 echo "Container ${CONTAINER_NAME} stopped"

templates/example-nginx/uninstall.sh

@@ -1,19 +1,13 @@
 #!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "CONTAINER_NAME"
 
-# UNINSTALL SCRIPT
+# Nginx Example Uninstall Script
-# The uninstall script is required for all templates.
-# It is used to uninstall the service from the server.
-# It is called with the path to the server specific env file as an argument.
 
-source "$(dirname "$0")/_common.sh"
+echo "Uninstalling service ${CONTAINER_NAME}..."
-check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG"
+_remove_container $CONTAINER_NAME || _die "Failed to remove container ${CONTAINER_NAME}"
+if _is_container_running $CONTAINER_NAME; then _die "Couldn't stop existing container"; fi
+if _is_container_exists $CONTAINER_NAME; then _die "Couldn't remove existing container"; fi
 
-_remove_container $CONTAINER_NAME || die "Failed to remove container ${CONTAINER_NAME}"
+echo "Service ${CONTAINER_NAME} uninstalled."
-_is_container_running && die "Couldn't stop existing container"
+echo "Note: This does NOT remove the local data folder. Use nuke.sh for that."
-_is_container_exists && die "Couldn't remove existing container"
-
-# remove the image
-docker rmi "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || echo "Failed to remove image $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG"
-
-echo "Uninstallation of ${CONTAINER_NAME} complete."
-echo "Local data folder ${LOCAL_DATA_FOLDER} still in place."

templates/simple-object-storage/README.txt

@@ -0,0 +1 @@
+simple-object-storage

templates/simple-object-storage/_default.env

@@ -0,0 +1,12 @@
+# Service settings specific to this server 
+
+# Image settings
+IMAGE_REGISTRY="gitea.jde.nz"
+IMAGE_REPO="j/simple-object-storage"
+IMAGE_TAG="latest"
+
+# Container settings
+CONTAINER_NAME="simple-object-storage"
+
+# Volume settings
+VOLUME_NAME="simple-object-storage"

templates/simple-object-storage/backup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+source "${AGENT_PATH}/_common.sh"
+_check_required_env_vars "VOLUME_NAME" "BACKUP_FILE" "TEMP_DIR"
+
+# Simple Object Storage Backup Script
+# Creates a backup tarball of the volume contents.
+
+
+
+# HOT backup is fine for simple-object-storage
+autobackup "volume=${VOLUME_NAME}" || _die "Failed to create backup"
+
+echo "Backup complete: ${BACKUP_FILE}"

templates/simple-object-storage/config/.template_info.env

@@ -0,0 +1,2 @@
+# Template to use - always required!
+TEMPLATE=simple-object-storage