Can now create local agent files.
Some checks failed
Dropshell Test / Build_and_Test (push) Has been cancelled

This commit is contained in:
Your Name 2025-05-17 10:12:57 +12:00
parent 49b1475ffd
commit 9eb9707c2e
8 changed files with 1144 additions and 736 deletions

View File

@ -25,7 +25,7 @@ string(TIMESTAMP RELEASE_DATE "%Y-%m-%d")
# Configure version.hpp file
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/src/version.hpp.in"
"${CMAKE_CURRENT_BINARY_DIR}/src/version.hpp"
"${CMAKE_CURRENT_BINARY_DIR}/src/autogen/version.hpp"
@ONLY
)
@ -56,6 +56,7 @@ target_include_directories(dropshell PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src/utils
${CMAKE_CURRENT_SOURCE_DIR}/src/contrib
${CMAKE_CURRENT_SOURCE_DIR}/src/commands
${CMAKE_CURRENT_SOURCE_DIR}/src/autogen
)
if(WIN32)

View File

@ -1,4 +1,5 @@
#!/bin/bash
set -e
# This script creates two files:
# src/utils/createagent.hpp
@ -7,123 +8,12 @@
SCRIPT_DIR=$(dirname "$0")
# check if dehydrate is installed
if ! command -v dehydrate &> /dev/null; then
echo "dehydrate could not be found - installing"
curl -fsSL https://gitea.jde.nz/public/dehydrate/releases/download/latest/install.sh | bash
fi
# heredoc to create the createagent.hpp file.
cat <<EOF > "$SCRIPT_DIR/src/utils/createagent.hpp"
#ifndef CREATEAGENT_HPP
#define CREATEAGENT_HPP
SCRIPT_DIR=$(dirname "$0")
/*
CREATEAGENT.HPP IS AUTOMATICALLY GENERATED FROM make_createagent.sh,
and recreates the files in /agent/ on a remote server.
DO NOT EDIT THIS FILE MANUALLY! Edit make_createagent.sh instead.
*/
#include <string>
namespace dropshell {
int create_agent(const std::string &server_name);
}
#endif // CREATEAGENT_HPP
EOF
# heredoc to create the createagent.cpp file.
cat <<CPPEOF > "$SCRIPT_DIR/src/utils/createagent.cpp"
#include "createagent.hpp"
#include "utils.hpp"
#include "directories.hpp"
#include "execute.hpp"
#include "server_env_manager.hpp"
#include <vector>
#include <string>
#include <iostream>
/*
CREATEAGENT.CPP IS AUTOMATICALLY GENERATED FROM make_createagent.sh,
and recreates the files in /agent/ on a remote server.
DO NOT EDIT THIS FILE MANUALLY! Edit make_createagent.sh instead.
*/
namespace dropshell {
struct AgentFile {
std::string filename;
std::string content;
};
int load_agent_file_contents(std::vector<AgentFile> &agent_files);
int remote_write_file(const sSSHInfo &ssh_info, const AgentFile &file);
int create_agent(const std::string &server_name) {
// create the sh file on the remote server.
server_env_manager server_env(server_name);
if (!server_env.is_valid()) {
std::cerr << "Invalid server environment for " << server_name << std::endl;
return 1;
}
// create the agent directory.
std::string agent_dir = remotepath::agent(server_name);
execute_ssh_command(server_env.get_SSH_INFO(), sCommand("","mkdir -p " + agent_dir,{}), cMode::Silent);
std::vector<AgentFile> agent_files;
load_agent_file_contents(agent_files);
// create the sh file on the remote server.
for (const auto &file : agent_files) {
// write out the file contents in the file.
if (remote_write_file(server_env.get_SSH_INFO(), file) != 0) {
std::cerr << "Failed to write file " << file.filename << std::endl;
return 1;
}
std::cout << "Wrote file " << file.filename << " on " << server_name << std::endl;
}
return 0;
}
int remote_write_file(const sSSHInfo &ssh_info, const AgentFile &file)
{
std::string generate_command = "cat <<'GENERATE_EOF' > " + file.filename + "\n" + file.content + "\nGENERATE_EOF\n"; // use heredoc to write the file.
return execute_ssh_command(ssh_info, sCommand("",generate_command,{}), cMode::Defaults);
}
// auto generated load_agent_file_contents function goes here, followed by closing } for namespace dropshell.
int load_agent_file_contents(std::vector<AgentFile> &agent_files)
{
agent_files={
CPPEOF
AGENT_SOURCE_FOLDER="$SCRIPT_DIR/agent"
# find all the files in the agent source folder.
find "$AGENT_SOURCE_FOLDER" -type f | while read -r file; do
# get the filename without the path.
filename=$(basename "$file")
echo " AgentFile { \"$filename\", R\"AGENTFILE(" >> "$SCRIPT_DIR/src/utils/createagent.cpp"
cat "$file" >> "$SCRIPT_DIR/src/utils/createagent.cpp"
echo ")AGENTFILE\" }," >> "$SCRIPT_DIR/src/utils/createagent.cpp"
done
# close the namespace dropshell.
cat <<CPPEOF >> "$SCRIPT_DIR/src/utils/createagent.cpp"
}; // agent_files defined.
return 0;
} // end of load_agent_file_contents function.
} // namespace dropshell
CPPEOF
dehydrate "${SCRIPT_DIR}/agent" "${SCRIPT_DIR}/src/autogen"

1111
src/autogen/_agent.cpp Normal file

File diff suppressed because it is too large Load Diff

15
src/autogen/_agent.hpp Normal file
View File

@ -0,0 +1,15 @@
#pragma once
/*
THIS FILE IS AUTO-GENERATED BY DEHYDRATE.
DO NOT EDIT THIS FILE.
*/
#include <string>
namespace recreate_agent {
bool recreate_tree(std::string destination_folder);
}

View File

@ -5,7 +5,7 @@
#include "templates.hpp"
#include "shared_commands.hpp"
#include "utils/hash.hpp"
#include "utils/createagent.hpp"
#include "autogen/_agent.hpp"
#include <unistd.h>
#include <cstring>
#include <iostream>
@ -276,6 +276,11 @@ namespace dropshell
std::cout << "Updating bb64..." << std::endl;
system((localpath::agent() + "bb64 -u").c_str()); // update.
}
std::cout << "Creating local agent files..." << std::endl;
recreate_agent::recreate_tree(localpath::agent());
return 0;
}
@ -330,7 +335,8 @@ namespace dropshell
std::cout << "Downloaded bb64 to " << agent_path << " on remote server." << std::endl;
// now create the agent.
create_agent(server);
// copy across from the local agent files.
#pragma message("TODO: copy across from the local agent files.")
return 0; // NOTIMPL
}

View File

@ -1,596 +0,0 @@
#include "createagent.hpp"
#include "utils.hpp"
#include "directories.hpp"
#include "execute.hpp"
#include "server_env_manager.hpp"
#include <vector>
#include <string>
#include <iostream>
/*
CREATEAGENT.CPP IS AUTOMATICALLY GENERATED FROM make_createagent.sh,
and recreates the files in /agent/ on a remote server.
DO NOT EDIT THIS FILE MANUALLY! Edit make_createagent.sh instead.
*/
namespace dropshell {
struct AgentFile {
std::string filename;
std::string content;
};
int load_agent_file_contents(std::vector<AgentFile> &agent_files);
int remote_write_file(const sSSHInfo &ssh_info, const AgentFile &file);
int create_agent(const std::string &server_name) {
// create the sh file on the remote server.
server_env_manager server_env(server_name);
if (!server_env.is_valid()) {
std::cerr << "Invalid server environment for " << server_name << std::endl;
return 1;
}
// create the agent directory.
std::string agent_dir = remotepath::agent(server_name);
execute_ssh_command(server_env.get_SSH_INFO(), sCommand("","mkdir -p " + agent_dir,{}), cMode::Silent);
std::vector<AgentFile> agent_files;
load_agent_file_contents(agent_files);
// create the sh file on the remote server.
for (const auto &file : agent_files) {
// write out the file contents in the file.
if (remote_write_file(server_env.get_SSH_INFO(), file) != 0) {
std::cerr << "Failed to write file " << file.filename << std::endl;
return 1;
}
std::cout << "Wrote file " << file.filename << " on " << server_name << std::endl;
}
return 0;
}
int remote_write_file(const sSSHInfo &ssh_info, const AgentFile &file)
{
std::string generate_command = "cat <<'GENERATE_EOF' > " + file.filename + "\n" + file.content + "\nGENERATE_EOF\n"; // use heredoc to write the file.
return execute_ssh_command(ssh_info, sCommand("",generate_command,{}), cMode::Defaults);
}
// auto generated load_agent_file_contents function goes here, followed by closing } for namespace dropshell.
int load_agent_file_contents(std::vector<AgentFile> &agent_files)
{
agent_files={
AgentFile { "_common.sh", R"AGENTFILE(
# COMMON FUNCTIONS
# JDE
# 2025-05-03
# This file is available TO ***ALL*** templates, as ${AGENT_PATH}/_common.sh
# ----------------------------------------------------------------------------------------------------------
# summary of functions:
# _die "message" : Prints an error message in red and exits with status code 1.
# _grey_start : Switches terminal output color to grey.
# _grey_end : Resets terminal output color from grey.
# _create_and_start_container "<run_cmd>" <container_name> : Creates/starts a container, verifying it runs.
# _create_folder <folder_path> : Creates a directory if it doesn't exist (chmod 777).
# _check_docker_installed : Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
# _is_container_exists <container_name> : Checks if a container (any state) exists. Returns 1 if not found.
# _is_container_running <container_name>: Checks if a container is currently running. Returns 1 if not running.
# _get_container_id <container_name> : Prints the ID of the named container.
# _get_container_status <container_name>: Prints the status string of the named container.
# _start_container <container_name> : Starts an existing, stopped container.
# _stop_container <container_name> : Stops a running container.
# _remove_container <container_name> : Stops (if needed) and removes a container.
# _get_container_logs <container_name> : Prints the logs for a container.
# _check_required_env_vars "VAR1" ... : Checks if listed environment variables are set; calls _die() if any are missing.
# _root_remove_tree <path> : Removes a path using a root Docker container (for permissions).
# ----------------------------------------------------------------------------------------------------------
# Prints an error message in red and exits with status code 1.
_die() {
echo -e "\033[91mError: $1\033[0m"
exit 1
}
# Switches terminal output color to grey.
_grey_start() {
echo -e -n "\033[90m"
}
# Resets terminal output color from grey.
_grey_end() {
echo -e -n "\033[0m"
}
# Creates/starts a container, verifying it runs.
_create_and_start_container() {
if [ -z "$1" ] || [ -z "$2" ]; then
_die "Template error: create_and_start_container <run_cmd> <container_name>"
fi
local run_cmd="$1"
local container_name="$2"
if _is_container_exists $container_name; then
_is_container_running $container_name && return 0
_start_container $container_name
else
_grey_start
$run_cmd
_grey_end
fi
if ! _is_container_running $container_name; then
_die "Container ${container_name} failed to start"
fi
ID=$(_get_container_id $container_name)
echo "Container ${container_name} is running with ID ${ID}"
}
# Creates a directory if it doesn't exist (chmod 777).
_create_folder() {
local folder="$1"
if [ -d "$folder" ]; then
return 0
fi
if ! mkdir -p "$folder"; then
_die "Failed to create folder: $folder"
fi
chmod 777 "$folder"
echo "Folder created: $folder"
}
# Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
_check_docker_installed() {
if ! command -v docker &> /dev/null; then
echo "Docker is not installed"
return 1
fi
# check if docker daemon is running
if ! docker info &> /dev/null; then
echo "Docker daemon is not running"
return 1
fi
# check if user has permission to run docker
if ! docker run --rm hello-world &> /dev/null; then
echo "User does not have permission to run docker"
return 1
fi
return 0
}
# Checks if a container (any state) exists. Returns 1 if not found.
_is_container_exists() {
if ! docker ps -a --format "{{.Names}}" | grep -q "^$1$"; then
return 1
fi
return 0
}
# Checks if a container is currently running. Returns 1 if not running.
_is_container_running() {
if ! docker ps --format "{{.Names}}" | grep -q "^$1$"; then
return 1
fi
return 0
}
# Prints the ID of the named container.
_get_container_id() {
docker ps --format "{{.ID}}" --filter "name=$1"
}
# Prints the status string of the named container.
_get_container_status() {
docker ps --format "{{.Status}}" --filter "name=$1"
}
# Starts an existing, stopped container.
_start_container() {
_is_container_exists $1 || return 1
_is_container_running $1 && return 0
docker start $1
}
# Stops a running container.
_stop_container() {
_is_container_running $1 || return 0;
docker stop $1
}
# Stops (if needed) and removes a container.
_remove_container() {
_stop_container $1
_is_container_exists $1 || return 0;
docker rm $1
}
# Prints the logs for a container.
_get_container_logs() {
if ! _is_container_exists $1; then
echo "Container $1 does not exist"
return 1
fi
docker logs $1
}
# Checks if listed environment variables are set; calls _die() if any are missing.
_check_required_env_vars() {
local required_vars=("$@")
for var in "${required_vars[@]}"; do
if [ -z "${!var}" ]; then
_die "Required environment variable $var is not set"
fi
done
}
# Removes a path using a root Docker container (for permissions).
_root_remove_tree() {
local to_remove="$1"
parent=$(dirname "$to_remove")
abs_parent=$(realpath "$parent")
child=$(basename "$to_remove")
docker run --rm -v "$abs_parent":/data alpine rm -rf "/data/$child"
}
# Load autocommands
source "${AGENT_PATH}/_autocommands.sh")AGENTFILE" },
AgentFile { "_nuke_other.sh", R"AGENTFILE(
#!/bin/bash
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "$SCRIPT_DIR/shared/_common.sh"
A_SERVICE="$1"
A_SERVICE_PATH="$2"
# 1. Check if service directory exists on server
[ -d "$A_SERVICE_PATH" ] || _die "Service is not installed: $A_SERVICE"
# uninstall the service
if [ -f "$A_SERVICE_PATH/uninstall.sh" ]; then
$A_SERVICE_PATH/uninstall.sh
fi
# nuke the service
if [ -f "$A_SERVICE_PATH/nuke.sh" ]; then
$A_SERVICE_PATH/nuke.sh
fi
# remove the service directory
rm -rf "$A_SERVICE_PATH"
)AGENTFILE" },
AgentFile { "_allservicesstatus.sh", R"AGENTFILE(
#!/bin/bash
# This script checks ALL services on the server and returns a status for each.
# Return format is simple ENV with the following format:
# SERVICE_NAME_HEALTH=healthy|unhealthy|unknown
# SERVICE_NAME_PORTS=port1,port2,port3
# Get all services on the server
SCRIPT_DIR="$(dirname "$0")"
# // DROPSHELL_DIR
# // |-- backups
# // |-- services
# // |-- service name
# // |-- config <-- this is passed as argument to all scripts
# // |-- service.env
# // |-- template
# // |-- (script files)
# // |-- shared
# // |-- _allservicesstatus.sh
# // |-- config
# // |-- service.env
# // |-- (other config files for specific server&service)
CURRENT_OUTPUT=""
CURRENT_EXIT_CODE=0
load_dotenv(){
local file_path=$1
if [ -f "${file_path}" ]; then
source "${file_path}"
fi
}
function run_command() {
local service_path=$1
local command=$2
local capture_output=${3:-false} # default to false if not specified
# check if the command is a file
if [ ! -f "${service_path}/template/${command}.sh" ]; then
return;
fi
# run the command in a subshell to prevent environment changes
CURRENT_OUTPUT=$(
set -a
load_dotenv "${service_path}/template/_default.env"
load_dotenv "${service_path}/config/service.env"
set +a
# update the main variables.
export CONFIG_PATH="${service_path}/config"
# SERVER is correct
export SERVICE="${SERVICE_NAME}"
if [ "$capture_output" = "true" ]; then
# Capture and return output
bash "${service_path}/template/${command}.sh" 2>&1
else
# Run silently and return exit code
bash "${service_path}/template/${command}.sh" > /dev/null 2>&1
fi
)
CURRENT_EXIT_CODE=$?
}
function command_exists() {
local service_path=$1
local command=$2
if [ ! -f "${service_path}/template/${command}.sh" ]; then
return 1
fi
return 0
}
# Get all services on the server
SERVICES_PATH=$(realpath "${SCRIPT_DIR}/../../../")
# Get all service names
SERVICE_NAMES=$(ls "${SERVICES_PATH}")
# Iterate over all service names
for SERVICE_NAME in ${SERVICE_NAMES}; do
SERVICE_PATH=$(realpath "${SERVICES_PATH}/${SERVICE_NAME}")
#--------------------------------
# Get the service health
if ! command_exists "${SERVICE_PATH}" "status"; then
SERVICE_HEALTH="unknown"
else
run_command "${SERVICE_PATH}" "status" "false"
if [ "${CURRENT_EXIT_CODE}" -eq 0 ]; then
SERVICE_HEALTH="healthy"
else
SERVICE_HEALTH="unhealthy"
fi
fi
#--------------------------------
# Get the service ports
if ! command_exists "${SERVICE_PATH}" "ports"; then
SERVICE_PORTS=""
else
run_command "${SERVICE_PATH}" "ports" "true"
SERVICE_PORTS="${CURRENT_OUTPUT}"
fi
#--------------------------------
# return the health and ports
echo "${SERVICE_NAME}_HEALTH=${SERVICE_HEALTH}"
echo "${SERVICE_NAME}_PORTS=${SERVICE_PORTS}"
done
)AGENTFILE" },
AgentFile { "_autocommands.sh", R"AGENTFILE(
#!/bin/bash
# This script contains the common code for the autocommands.
MYID=$(id -u)
MYGRP=$(id -g)
_autocommandrun_volume() {
local command="$1"
local volume_name="$2"
local backup_folder="$3"
case "$command" in
create)
echo "Creating volume ${volume_name}"
docker volume create ${volume_name}
;;
nuke)
echo "Nuking volume ${volume_name}"
docker volume rm ${volume_name}
;;
backup)
echo "Backing up volume ${volume_name}"
docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /volume . && chown -R $MYID:$MYGRP /backup"
;;
restore)
echo "Restoring volume ${volume_name}"
docker volume rm ${volume_name}
docker volume create ${volume_name}
docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -xzvf /backup/backup.tgz -C /volume --strip-components=1"
;;
esac
}
_autocommandrun_path() {
local command="$1"
local path="$2"
local backup_folder="$3"
case "$command" in
create)
echo "Creating path ${path}"
mkdir -p ${path}
;;
nuke)
echo "Nuking path ${path}"
local path_parent=$(dirname ${path})
local path_child=$(basename ${path})
if [ -d "${path_parent}/${path_child}" ]; then
docker run --rm -v ${path_parent}:/volume debian bash -c "rm -rf /volume/${path_child}" || echo "Failed to nuke path ${path}"
else
echo "Path ${path} does not exist - nothing to nuke"
fi
;;
backup)
echo "Backing up path ${path}"
if [ -d "${path}" ]; then
docker run --rm -v ${path}:/path -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /path . && chown -R $MYID:$MYGRP /backup"
else
echo "Path ${path} does not exist - nothing to backup"
fi
;;
restore)
echo "Restoring path ${path}"
tar -xzvf ${backup_folder}/backup.tgz -C ${path} --strip-components=1
;;
esac
}
_autocommandrun_file() {
local command="$1"
local filepath="$2"
local backup_folder="$3"
case "$command" in
create)
;;
nuke)
rm -f ${filepath}
;;
backup)
echo "Backing up file ${filepath}"
local file_parent=$(dirname ${filepath})
local file_name=$(basename ${filepath})
if [ -f "${file_parent}/${file_name}" ]; then
docker run --rm -v ${file_parent}:/volume -v ${backup_folder}:/backup debian bash -c "cp /volume/${file_name} /backup/${file_name} && chown -R $MYID:$MYGRP /backup"
else
echo "File ${filepath} does not exist - nothing to backup"
fi
;;
restore)
echo "Restoring file ${filepath}"
local file_name=$(basename ${filepath})
cp ${backup_folder}/${file_name} ${filepath}
;;
esac
}
_autocommandparse() {
# first argument is the command
# if the command is backup or restore, then the last two arguments are the backup file and the temporary path
# all other arguments are of form:
# key=value
# where key can be one of volume, path or file.
# value is the path or volume name.
# we iterate over the key=value arguments, and for each we call:
# autorun <command> <backupfile> <key> <value>
local command="$1"
shift
local backup_temp_path="$1"
shift
echo "autocommandparse: command=$command backup_temp_path=$backup_temp_path"
# Extract the backup file and temp path (last two arguments)
local args=("$@")
local arg_count=${#args[@]}
# Process all key=value pairs
for ((i=0; i<$arg_count; i++)); do
local pair="${args[$i]}"
# Skip if not in key=value format
if [[ "$pair" != *"="* ]]; then
continue
fi
local key="${pair%%=*}"
local value="${pair#*=}"
# create backup folder unique to key/value.
local bfolder=$(echo "${key}_${value}" | tr -cd '[:alnum:]_-')
local targetpath="${backup_temp_path}/${bfolder}"
mkdir -p ${targetpath}
# Key must be one of volume, path or file
case "$key" in
volume)
_autocommandrun_volume "$command" "$value" "$targetpath"
;;
path)
_autocommandrun_path "$command" "$value" "$targetpath"
;;
file)
_autocommandrun_file "$command" "$value" "$targetpath"
;;
*)
_die "Unknown key $key passed to auto${command}. We only support volume, path and file."
;;
esac
done
}
autocreate() {
_autocommandparse create none "$@"
}
autonuke() {
_autocommandparse nuke none "$@"
}
autobackup() {
_check_required_env_vars "BACKUP_FILE" "TEMP_DIR"
BACKUP_TEMP_PATH="$TEMP_DIR/backup"
mkdir -p "$BACKUP_TEMP_PATH"
echo "_autocommandparse [backup] [$BACKUP_TEMP_PATH] [$@]"
_autocommandparse backup "$BACKUP_TEMP_PATH" "$@"
tar zcvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" .
}
autorestore() {
_check_required_env_vars "BACKUP_FILE" "TEMP_DIR"
BACKUP_TEMP_PATH="$TEMP_DIR/restore"
echo "_autocommandparse [restore] [$BACKUP_TEMP_PATH] [$@]"
mkdir -p "$BACKUP_TEMP_PATH"
tar zxvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" --strip-components=1
_autocommandparse restore "$BACKUP_TEMP_PATH" "$@"
}
)AGENTFILE" },
}; // agent_files defined.
return 0;
} // end of load_agent_file_contents function.
} // namespace dropshell

View File

@ -1,19 +0,0 @@
#ifndef CREATEAGENT_HPP
#define CREATEAGENT_HPP
/*
CREATEAGENT.HPP IS AUTOMATICALLY GENERATED FROM make_createagent.sh,
and recreates the files in /agent/ on a remote server.
DO NOT EDIT THIS FILE MANUALLY! Edit make_createagent.sh instead.
*/
#include <string>
namespace dropshell {
int create_agent(const std::string &server_name);
}
#endif // CREATEAGENT_HPP