dropshell release 2025.0519.0021

source/agent-remote/_allservicesstatus.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# This script checks ALL services on the server and returns a status for each.
+
+# Return format is simple ENV with the following format:
+# SERVICE_NAME_HEALTH=healthy|unhealthy|unknown
+# SERVICE_NAME_PORTS=port1,port2,port3
+
+# Get all services on the server
+SCRIPT_DIR="$(dirname "$0")"
+
+    # //------------------------------------------------------------------------------------------------
+    # // remote paths
+    # // DROPSHELL_DIR
+    # //   |-- backups
+    # //   |-- temp_files
+    # //   |-- agent
+    # //   |   |-- bb64
+    # //   |   |-- (other agent files, including _allservicesstatus.sh)
+    # //   |-- services
+    # //       |-- service name
+    # //           |-- config
+    # //               |-- service.env (actual service config)
+    # //               |-- .template_info.env
+    # //           |-- template
+    # //               |-- _default.env
+    # //               |-- (script files)
+    # //               |-- config
+    # //                   |-- service.env (default service config)
+    # //                   |-- .template_info.env
+    # //                   |-- (other config files for specific server&service)
+
+# Get all services on the server
+SERVICES_PATH=$(realpath "${SCRIPT_DIR}/../services/")
+
+CURRENT_OUTPUT=""
+CURRENT_EXIT_CODE=0
+
+load_dotenv(){
+    local file_path=$1
+    if [ -f "${file_path}" ]; then
+        source "${file_path}"
+    fi
+}
+
+_check_required_env_vars_allservicesstatus() {
+    local required_vars=("$@")
+    for var in "${required_vars[@]}"; do
+        if [ -z "${!var}" ]; then
+            _die "Required environment variable $var is not set"
+        fi
+    done
+}
+
+function run_command() {
+    local service_path=$1
+    local command=$2
+    local capture_output=${3:-false}  # default to false if not specified
+
+    # check if the command is a file
+    if [ ! -f "${service_path}/template/${command}.sh" ]; then
+        return;
+    fi
+
+    # run the command in a subshell to prevent environment changes
+    CURRENT_OUTPUT=$(
+        set -a
+
+        load_dotenv "${service_path}/template/_default.env"
+        load_dotenv "${service_path}/config/service.env"
+        load_dotenv "${service_path}/config/.template_info.env"
+
+        # update the main variables.
+        CONFIG_PATH="${service_path}/config"
+        SERVICE="${SERVICE_NAME}"
+        DOCKER_CLI_HINTS=false
+
+        set +a
+
+        _check_required_env_vars_allservicesstatus "CONFIG_PATH" "SERVER" "SERVICE" "AGENT_PATH" "HOST_NAME" "TEMPLATE"
+
+        if [ "$capture_output" = "true" ]; then
+            # Capture and return output
+            bash "${service_path}/template/${command}.sh" 2>&1
+        else
+            # Run silently and return exit code
+            bash "${service_path}/template/${command}.sh" > /dev/null 2>&1
+        fi
+    )
+    CURRENT_EXIT_CODE=$?
+}
+
+function command_exists() {
+    local service_path=$1
+    local command=$2
+    if [ ! -f "${service_path}/template/${command}.sh" ]; then
+        return 1
+    fi
+    return 0
+}
+
+
+if [ ! -d "${SERVICES_PATH}" ]; then
+    echo "Services path does not exist: ${SERVICES_PATH}"
+    exit 0
+fi
+
+# Get all service names
+SERVICE_NAMES=$(ls "${SERVICES_PATH}")
+
+# Iterate over all service names
+for SERVICE_NAME in ${SERVICE_NAMES}; do
+
+    SERVICE_PATH=$(realpath "${SERVICES_PATH}/${SERVICE_NAME}")
+
+    #--------------------------------
+    # Get the service health
+    if ! command_exists "${SERVICE_PATH}" "status"; then
+        SERVICE_HEALTH="unknown"
+    else
+        run_command "${SERVICE_PATH}" "status" "false"
+        if [ "${CURRENT_EXIT_CODE}" -eq 0 ]; then
+            SERVICE_HEALTH="healthy"
+        else
+            SERVICE_HEALTH="unhealthy"
+        fi
+    fi
+
+    #--------------------------------
+    # Get the service ports
+    if ! command_exists "${SERVICE_PATH}" "ports"; then
+        SERVICE_PORTS=""
+    else
+        run_command "${SERVICE_PATH}" "ports" "true"
+        SERVICE_PORTS="${CURRENT_OUTPUT}"
+    fi
+
+    #--------------------------------
+    # return the health and ports
+    echo "${SERVICE_NAME}_HEALTH=${SERVICE_HEALTH}"
+    echo "${SERVICE_NAME}_PORTS=${SERVICE_PORTS}"
+done

source/agent-remote/agent-install.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# This script is used to install the dropshell agent on a remote server.
+
+SCRIPT_DIR=$(dirname "$0")
+
+set -a
+
+AGENT_PATH="$SCRIPT_DIR"
+
+set +a
+
+
+if [ -f "$SCRIPT_DIR/common.sh" ]; then
+    source "$SCRIPT_DIR/common.sh"
+else
+    echo "Error: common.sh not found in $SCRIPT_DIR"
+    exit 1
+fi
+
+_check_required_env_vars "AGENT_PATH"
+
+function install_bb64() {    
+    # check curl installed
+    if ! command -v curl &> /dev/null; then
+        _die "Curl is not installed. Curl is required for agent installation."
+    fi
+
+    curl -fsSL "https://gitea.jde.nz/public/bb64/releases/download/latest/install.sh" | bash -s -- "$AGENT_PATH" "$(id -u $USER):$(id -g $USER)"
+
+    # test result code from curl
+    if [ $? -ne 0 ]; then
+        _die "Failed to install bb64. Curl returned non-zero exit code."
+    fi
+
+    # test if bb64 is installed
+    "$AGENT_PATH/bb64" -v
+    if [ $? -ne 0 ]; then
+        _die "bb64 did not install correctly."
+    fi
+
+    echo "bb64 installed successfully."
+    return 0;
+}
+
+#-------------------------------------------------------------------------
+
+echo "Installing dropshell agent..."
+
+install_bb64
+
+
+
+#-------------------------------------------------------------------------
+
+echo "Running remote agent self-test..."
+
+
+#-------------------------------------------------------------------------
+
+echo "Completed remote agent self-test."
+
+
+#-------------------------------------------------------------------------
+
+echo "Completed dropshell agent installation."
+
+#-------------------------------------------------------------------------
+
+exit 0

source/agent-remote/common.sh

@@ -0,0 +1,171 @@
+# COMMON FUNCTIONS
+# JDE
+# 2025-05-03
+
+# This file is available TO ***ALL*** templates, as ${AGENT_PATH}/_common.sh
+
+# ----------------------------------------------------------------------------------------------------------
+
+# summary of functions:
+#   _die "message"                         : Prints an error message in red and exits with status code 1.
+#   _grey_start                            : Switches terminal output color to grey.
+#   _grey_end                              : Resets terminal output color from grey.
+#   _create_and_start_container "<run_cmd>" <container_name> : Creates/starts a container, verifying it runs.
+#   _create_folder <folder_path>           : Creates a directory if it doesn't exist (chmod 777).
+#   _check_docker_installed               : Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
+#   _is_container_exists <container_name> : Checks if a container (any state) exists. Returns 1 if not found.
+#   _is_container_running <container_name>: Checks if a container is currently running. Returns 1 if not running.
+#   _get_container_id <container_name>    : Prints the ID of the named container.
+#   _get_container_status <container_name>: Prints the status string of the named container.
+#   _start_container <container_name>     : Starts an existing, stopped container.
+#   _stop_container <container_name>      : Stops a running container.
+#   _remove_container <container_name>    : Stops (if needed) and removes a container.
+#   _get_container_logs <container_name>  : Prints the logs for a container.
+#   _check_required_env_vars "VAR1" ...    : Checks if listed environment variables are set; calls _die() if any are missing.
+#   _root_remove_tree <path>              : Removes a path using a root Docker container (for permissions).
+
+# ----------------------------------------------------------------------------------------------------------
+
+# Prints an error message in red and exits with status code 1.
+_die() {
+    echo -e "Error: $1"
+    exit 1
+}
+
+# Creates/starts a container, verifying it runs.
+_create_and_start_container() {
+    if [ -z "$1" ] || [ -z "$2" ]; then
+        _die "Template error: create_and_start_container <run_cmd> <container_name>"
+    fi
+
+    local run_cmd="$1"
+    local container_name="$2"
+
+    if _is_container_exists $container_name; then
+        _is_container_running $container_name && return 0
+        _start_container $container_name
+    else
+        $run_cmd
+    fi
+
+    if ! _is_container_running $container_name; then
+        _die "Container ${container_name} failed to start"
+    fi
+
+    ID=$(_get_container_id $container_name)
+    echo "Container ${container_name} is running with ID ${ID}"
+}
+
+# Creates a directory if it doesn't exist (chmod 777).
+_create_folder() {
+    local folder="$1"
+    if [ -d "$folder" ]; then
+        return 0
+    fi
+    if ! mkdir -p "$folder"; then
+        _die "Failed to create folder: $folder"
+    fi
+    chmod 777 "$folder"
+    echo "Folder created: $folder"
+}
+
+# Checks if Docker is installed, running, and user has permission. Returns 1 on failure.
+_check_docker_installed() {
+    if ! command -v docker &> /dev/null; then
+        echo "Docker is not installed"
+        return 1
+    fi
+
+    # check if docker daemon is running
+    if ! docker info &> /dev/null; then
+        echo "Docker daemon is not running"
+        return 1
+    fi
+
+    # check if user has permission to run docker
+    if ! docker run --rm hello-world &> /dev/null; then
+        echo "User does not have permission to run docker"
+        return 1
+    fi
+
+    return 0
+}
+
+# Checks if a container (any state) exists. Returns 1 if not found.
+_is_container_exists() {
+    if ! docker ps -a --format "{{.Names}}" | grep -q "^$1$"; then
+        return 1
+    fi
+    return 0
+}
+
+# Checks if a container is currently running. Returns 1 if not running.
+_is_container_running() {
+    if ! docker ps --format "{{.Names}}" | grep -q "^$1$"; then
+        return 1
+    fi
+    return 0
+}
+
+# Prints the ID of the named container.
+_get_container_id() {
+    docker ps --format "{{.ID}}" --filter "name=$1"
+}
+
+# Prints the status string of the named container.
+_get_container_status() {
+    docker ps --format "{{.Status}}" --filter "name=$1"
+}
+
+# Starts an existing, stopped container.
+_start_container() {
+    _is_container_exists $1 || return 1
+    _is_container_running $1 && return 0
+    docker start $1
+}
+
+# Stops a running container.
+_stop_container() {
+    _is_container_running $1 || return 0;
+    docker stop $1
+}   
+
+# Stops (if needed) and removes a container.
+_remove_container() {
+    _stop_container $1
+    _is_container_exists $1 || return 0;
+    docker rm $1
+}
+
+# Prints the logs for a container.
+_get_container_logs() {
+    if ! _is_container_exists $1; then
+        echo "Container $1 does not exist"
+        return 1
+    fi
+
+    docker logs $1
+}
+
+# Checks if listed environment variables are set; calls _die() if any are missing.
+_check_required_env_vars() {
+    local required_vars=("$@")
+    for var in "${required_vars[@]}"; do
+        if [ -z "${!var}" ]; then
+            _die "Required environment variable $var is not set"
+        fi
+    done
+}
+
+# Removes a path using a root Docker container (for permissions).
+_root_remove_tree() {
+    local to_remove="$1"
+    parent=$(dirname "$to_remove")
+    abs_parent=$(realpath "$parent")
+    child=$(basename "$to_remove")
+    docker run --rm -v "$abs_parent":/data alpine rm -rf "/data/$child"
+}
+
+
+# Load autocommands
+source "${AGENT_PATH}/datacommands.sh"

source/agent-remote/datacommands.sh

@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# This script contains the common code for the autocommands.
+
+MYID=$(id -u)
+MYGRP=$(id -g)
+
+_autocommandrun_volume() {
+    local command="$1"
+    local volume_name="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            echo "Creating volume ${volume_name}"
+            docker volume create ${volume_name}
+            ;;
+        nuke)
+            echo "Nuking volume ${volume_name}"
+            docker volume rm ${volume_name}
+            ;;
+        backup)
+            echo "Backing up volume ${volume_name}"
+            docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /volume . && chown -R $MYID:$MYGRP /backup"
+            ;;
+        restore)
+            echo "Restoring volume ${volume_name}"
+            docker volume rm ${volume_name}
+            docker volume create ${volume_name}
+            docker run --rm -v ${volume_name}:/volume -v ${backup_folder}:/backup debian bash -c "tar -xzvf /backup/backup.tgz -C /volume --strip-components=1"
+            ;;
+    esac
+}   
+
+_autocommandrun_path() {
+    local command="$1"
+    local path="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            echo "Creating path ${path}"
+            mkdir -p ${path}
+            ;;
+        nuke)
+            echo "Nuking path ${path}"
+            local path_parent=$(dirname ${path})
+            local path_child=$(basename ${path})
+            if [ -d "${path_parent}/${path_child}" ]; then
+                docker run --rm -v ${path_parent}:/volume debian bash -c "rm -rfv /volume/${path_child}" || echo "Failed to nuke path ${path}"
+            else
+                echo "Path ${path} does not exist - nothing to nuke"
+            fi
+            ;;
+        backup)
+            echo "Backing up path ${path}"
+            if [ -d "${path}" ]; then
+                docker run --rm -v ${path}:/path -v ${backup_folder}:/backup debian bash -c "tar -czvf /backup/backup.tgz -C /path . && chown -R $MYID:$MYGRP /backup"
+            else
+                echo "Path ${path} does not exist - nothing to backup"
+            fi
+            ;;
+        restore)
+            if [ ! -f "${backup_folder}/backup.tgz" ]; then
+                echo "Backup file ${backup_folder}/backup.tgz does not exist - nothing to restore"
+            else
+                echo "Clearing existing data in path ${path}"
+                docker run --rm -v ${path}:/path debian bash -c "rm -rfv /path/{*,.*}"
+                echo "Restoring path ${path} from backup file ${backup_folder}/backup.tgz"
+                tar -xzvf ${backup_folder}/backup.tgz -C ${path} --strip-components=1
+            fi
+            ;;
+    esac
+}
+
+_autocommandrun_file() {
+    local command="$1"
+    local filepath="$2"
+    local backup_folder="$3"
+
+    case "$command" in
+        create)
+            ;;
+        nuke)
+            rm -f ${filepath}
+            ;;
+        backup)
+            echo "Backing up file ${filepath}"
+            local file_parent=$(dirname ${filepath})
+            local file_name=$(basename ${filepath})
+            if [ -f "${file_parent}/${file_name}" ]; then
+                docker run --rm -v ${file_parent}:/volume -v ${backup_folder}:/backup debian bash -c "cp /volume/${file_name} /backup/${file_name} && chown -R $MYID:$MYGRP /backup"
+            else
+                echo "File ${filepath} does not exist - nothing to backup"
+            fi
+            ;;
+        restore)
+            echo "Restoring file ${filepath}"
+            local file_name=$(basename ${filepath})
+            rm -f ${filepath} || die "Unable to remove existing file ${filepath}, restore failed."
+            cp ${backup_folder}/${file_name} ${filepath} || die "Unable to copy file ${backup_folder}/${file_name} to ${filepath}, restore failed."
+            ;;
+    esac
+}
+
+_autocommandparse() {
+    # first argument is the command
+    # if the command is backup or restore, then the last two arguments are the backup file and the temporary path
+    # all other arguments are of form:
+    # key=value
+    # where key can be one of volume, path or file.
+    # value is the path or volume name.
+
+    # we iterate over the key=value arguments, and for each we call:
+    #    autorun <command> <backupfile> <key> <value>
+
+    local command="$1"
+    shift
+
+    local backup_temp_path="$1"
+    shift
+
+    echo "autocommandparse: command=$command backup_temp_path=$backup_temp_path"
+
+    # Extract the backup file and temp path (last two arguments)
+    local args=("$@")
+    local arg_count=${#args[@]}
+    
+    # Process all key=value pairs
+    for ((i=0; i<$arg_count; i++)); do
+        local pair="${args[$i]}"
+        
+        # Skip if not in key=value format
+        if [[ "$pair" != *"="* ]]; then
+            continue
+        fi
+        
+        local key="${pair%%=*}"
+        local value="${pair#*=}"
+
+        # create backup folder unique to key/value.
+        local bfolder=$(echo "${key}_${value}" | tr -cd '[:alnum:]_-')
+        local targetpath="${backup_temp_path}/${bfolder}"
+        mkdir -p ${targetpath}
+
+        # Key must be one of volume, path or file
+        case "$key" in
+            volume)
+                _autocommandrun_volume "$command" "$value" "$targetpath"
+                ;;
+            path)
+                _autocommandrun_path "$command" "$value" "$targetpath"
+                ;;
+            file)
+                _autocommandrun_file "$command" "$value" "$targetpath"
+                ;;
+            *)
+                _die "Unknown key $key passed to auto${command}. We only support volume, path and file."
+                ;;
+        esac
+    done
+}
+
+
+datacreate() {
+    _autocommandparse create none "$@"
+}
+
+
+datanuke() {
+    _autocommandparse nuke none "$@"
+}
+
+databackup() {
+    _check_required_env_vars "BACKUP_FILE" "TEMP_DIR"
+    BACKUP_TEMP_PATH="$TEMP_DIR/backup"
+
+
+    mkdir -p "$BACKUP_TEMP_PATH"
+    echo "_autocommandparse [backup] [$BACKUP_TEMP_PATH] [$@]"
+    _autocommandparse backup "$BACKUP_TEMP_PATH" "$@"
+
+    tar zcvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" .
+}
+
+datarestore() {
+    _check_required_env_vars "BACKUP_FILE" "TEMP_DIR"
+    BACKUP_TEMP_PATH="$TEMP_DIR/restore"
+
+    echo "_autocommandparse [restore] [$BACKUP_TEMP_PATH] [$@]"
+
+    mkdir -p "$BACKUP_TEMP_PATH"
+    tar zxvf "$BACKUP_FILE" -C "$BACKUP_TEMP_PATH" --strip-components=1
+
+    _autocommandparse restore "$BACKUP_TEMP_PATH" "$@"
+}