significant tidying to logserver

logclient/_volumes.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Define volume items for logclient container
+# These are used across backup, restore, create, and destroy operations
+
+get_logclient_volumes() {
+    echo "volume:datavolume:${DATA_VOLUME}" "volume:configvolume:${CONFIG_VOLUME}"
+}

logclient/backup.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+echo "Backing up ${CONTAINER_NAME} volumes..."
+
+# Stop the container to ensure data consistency
+bash ./stop.sh || true
+
+# Backup volumes
+BACKUP_DIR="${CONFIG_PATH}/backups/$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+
+# Export volumes
+for volume in $(get_logclient_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    echo "Backing up volume: $volume_name"
+    docker run --rm -v "$volume_name:/source:ro" -v "$BACKUP_DIR:/backup" alpine \
+        tar -czf "/backup/${volume_name}.tar.gz" -C /source .
+done
+
+# Backup configuration
+cp -r "${CONFIG_PATH}" "$BACKUP_DIR/config_backup"
+
+echo "Backup completed to: $BACKUP_DIR"
+
+# Restart the container
+bash ./start.sh
+
+echo "Container restarted"

logclient/config/.template_info.env

@@ -12,6 +12,5 @@ IMAGE_REPO="beats/filebeat"
 IMAGE_TAG="7.17.23"
 
 # Volume definitions
-CONFIG_VOLUME="${CONTAINER_NAME}_config"
-DATA_VOLUME="${CONTAINER_NAME}_data"
-CERTS_VOLUME="${CONTAINER_NAME}_certs"
+DATA_VOLUME=logclient_data
+CONFIG_VOLUME=logclient_config

logclient/destroy.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+echo "WARNING: This will permanently delete all data for ${CONTAINER_NAME}"
+echo "This action cannot be undone!"
+echo ""
+
+# Since scripts must be non-interactive, only proceed if explicitly called
+bash ./stop.sh || true
+_remove_container "$CONTAINER_NAME" || true
+
+# Remove all volumes
+for volume in $(get_logclient_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    echo "Removing volume: $volume_name"
+    docker volume rm "$volume_name" 2>/dev/null || true
+done
+
+echo "Service and all data destroyed"

logclient/install.sh

@@ -39,22 +39,10 @@ echo "Generating Filebeat configuration..."
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 bash "$SCRIPT_DIR/scripts/generate-config.sh" || _die "Failed to generate configuration"
 
-# Create Docker volumes
-CONFIG_VOLUME="${CONFIG_VOLUME:-${CONTAINER_NAME}_config}"
-DATA_VOLUME="${DATA_VOLUME:-${CONTAINER_NAME}_data}"
-CERTS_VOLUME="${CERTS_VOLUME:-${CONTAINER_NAME}_certs}"
-
-echo "Creating Docker volumes..."
-docker volume create "$CONFIG_VOLUME" >/dev/null 2>&1 || true
-docker volume create "$DATA_VOLUME" >/dev/null 2>&1 || true
-docker volume create "$CERTS_VOLUME" >/dev/null 2>&1 || true
-
-# Copy config to volume
-if [ -f "${CONFIG_PATH}/filebeat.yml" ]; then
-    echo "Copying configuration to Docker volume..."
-    docker run --rm -v "${CONFIG_VOLUME}:/config" -v "${CONFIG_PATH}:/source:ro" alpine \
-        cp /source/filebeat.yml /config/filebeat.yml
-fi
+# Create volumes using common function
+source "$SCRIPT_DIR/_volumes.sh"
+echo "Creating volumes..."
+create_items $(get_logclient_volumes)
 
 # Start the new container
 bash ./start.sh || _die "Failed to start Filebeat"

logclient/restore.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <backup_directory>"
+    echo "Available backups:"
+    ls -la "${CONFIG_PATH}/backups/" 2>/dev/null || echo "No backups found"
+    exit 1
+fi
+
+BACKUP_DIR="$1"
+
+if [ ! -d "$BACKUP_DIR" ]; then
+    _die "Backup directory not found: $BACKUP_DIR"
+fi
+
+echo "Restoring from backup: $BACKUP_DIR"
+
+# Stop the container
+bash ./stop.sh || true
+
+# Restore volumes
+for volume in $(get_logclient_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    backup_file="$BACKUP_DIR/${volume_name}.tar.gz"
+
+    if [ -f "$backup_file" ]; then
+        echo "Restoring volume: $volume_name"
+        docker run --rm -v "$volume_name:/target" -v "$BACKUP_DIR:/backup:ro" alpine \
+            sh -c "rm -rf /target/* && tar -xzf /backup/${volume_name}.tar.gz -C /target"
+    fi
+done
+
+# Restore configuration if exists
+if [ -d "$BACKUP_DIR/config_backup" ]; then
+    cp -r "$BACKUP_DIR/config_backup/"* "${CONFIG_PATH}/"
+fi
+
+echo "Restore completed"
+
+# Start the container
+bash ./start.sh
+
+echo "Container restarted"

logclient/start.sh

@@ -2,21 +2,20 @@
 source "${AGENT_PATH}/common.sh"
 _check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG"
 
-# Define volume names if not set
-CONFIG_VOLUME="${CONFIG_VOLUME:-${CONTAINER_NAME}_config}"
-DATA_VOLUME="${DATA_VOLUME:-${CONTAINER_NAME}_data}"
-CERTS_VOLUME="${CERTS_VOLUME:-${CONTAINER_NAME}_certs}"
+# Check that config file exists
+if [ ! -f "${CONFIG_PATH}/filebeat.yml" ]; then
+    _die "filebeat.yml not found in ${CONFIG_PATH}/filebeat.yml"
+fi
 
 # Create Docker command
-cmd="docker run -d \
+DOCKER_RUN_CMD="docker run -d \
     --name $CONTAINER_NAME \
     --restart unless-stopped \
     --user root \
     -v /var/run/docker.sock:/var/run/docker.sock:ro \
     -v /var/log:/var/log:ro \
-    -v ${CONFIG_VOLUME}:/usr/share/filebeat/config:ro \
+    -v ${CONFIG_PATH}:/usr/share/filebeat/config:ro \
     -v ${DATA_VOLUME}:/usr/share/filebeat/data \
-    -v ${CERTS_VOLUME}:/usr/share/filebeat/certs:ro \
     -e LOGSERVER_HOST=${LOGSERVER_HOST} \
     -e LOGSERVER_PORT=${LOGSERVER_PORT} \
     -e API_KEY=${API_KEY} \
@@ -24,7 +23,13 @@ cmd="docker run -d \
     filebeat -e -strict.perms=false \
     -c /usr/share/filebeat/config/filebeat.yml"
 
-# Start the container
-eval "$cmd" || _die "Failed to start Filebeat container"
+if ! _create_and_start_container "$DOCKER_RUN_CMD" "$CONTAINER_NAME"; then
+    _die "Failed to start container ${CONTAINER_NAME}"
+fi
 
-echo "Filebeat started successfully"
+# Check if the container is running
+if ! _is_container_running "$CONTAINER_NAME"; then
+    _die "Container ${CONTAINER_NAME} is not running"
+fi
+
+echo "Container ${CONTAINER_NAME} started"

logserver/_volumes.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Define volume items for logserver container
+# These are used across backup, restore, create, and destroy operations
+
+get_logserver_volumes() {
+    echo "volume:elasticsearch_data:${CONTAINER_NAME}_elasticsearch_data" \
+         "volume:logstash_data:${CONTAINER_NAME}_logstash_data" \
+         "volume:kibana_data:${CONTAINER_NAME}_kibana_data"
+}

logserver/backup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+echo "Backing up ${CONTAINER_NAME} ELK stack volumes..."
+echo "Note: This may take a while for large log databases"
+
+# Stop the containers to ensure data consistency
+bash ./stop.sh || true
+
+# Backup volumes
+BACKUP_DIR="${CONFIG_PATH}/backups/$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+
+# Export volumes
+for volume in $(get_logserver_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    echo "Backing up volume: $volume_name"
+    docker run --rm -v "$volume_name:/source:ro" -v "$BACKUP_DIR:/backup" alpine \
+        tar -czf "/backup/${volume_name}.tar.gz" -C /source .
+done
+
+# Backup configuration
+cp -r "${CONFIG_PATH}" "$BACKUP_DIR/config_backup"
+
+# Backup docker-compose.yml
+cp docker-compose.yml "$BACKUP_DIR/" 2>/dev/null || true
+
+echo "Backup completed to: $BACKUP_DIR"
+echo "Size: $(du -sh $BACKUP_DIR | cut -f1)"
+
+# Restart the containers
+bash ./start.sh
+
+echo "ELK stack restarted"

logserver/destroy.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+echo "WARNING: This will PERMANENTLY DELETE all ELK stack data!"
+echo "This includes all logs, dashboards, and configurations"
+echo "This action cannot be undone!"
+echo ""
+
+# Since scripts must be non-interactive, only proceed if explicitly called
+# Stop all containers
+docker compose down || true
+
+# Remove all volumes
+for volume in $(get_logserver_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    echo "Removing volume: $volume_name"
+    docker volume rm "$volume_name" 2>/dev/null || true
+done
+
+# Also remove the Docker network if it exists
+docker network rm "${CONTAINER_NAME}_elk" 2>/dev/null || true
+
+echo "ELK stack and all data destroyed"

logserver/install.sh

@@ -31,6 +31,11 @@ docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} || _die
 docker pull docker.elastic.co/logstash/logstash:${LS_VERSION} || _die "Failed to pull Logstash"
 docker pull docker.elastic.co/kibana/kibana:${KIBANA_VERSION} || _die "Failed to pull Kibana"
 
+# Create volumes using common function
+source "$SCRIPT_DIR/_volumes.sh"
+echo "Creating volumes..."
+create_items $(get_logserver_volumes)
+
 # Ensure config directory exists
 mkdir -p "${CONFIG_PATH}"
 

logserver/restore.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+source "${AGENT_PATH}/common.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh"
+
+_check_required_env_vars "CONTAINER_NAME"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <backup_directory>"
+    echo "Available backups:"
+    ls -la "${CONFIG_PATH}/backups/" 2>/dev/null || echo "No backups found"
+    exit 1
+fi
+
+BACKUP_DIR="$1"
+
+if [ ! -d "$BACKUP_DIR" ]; then
+    _die "Backup directory not found: $BACKUP_DIR"
+fi
+
+echo "Restoring ELK stack from backup: $BACKUP_DIR"
+echo "WARNING: This will overwrite all current data!"
+
+# Stop the containers
+bash ./stop.sh || true
+
+# Restore volumes
+for volume in $(get_logserver_volumes); do
+    volume_name=$(echo $volume | cut -d: -f3)
+    backup_file="$BACKUP_DIR/${volume_name}.tar.gz"
+
+    if [ -f "$backup_file" ]; then
+        echo "Restoring volume: $volume_name"
+        # Clear existing data and restore
+        docker run --rm -v "$volume_name:/target" -v "$BACKUP_DIR:/backup:ro" alpine \
+            sh -c "rm -rf /target/* && tar -xzf /backup/${volume_name}.tar.gz -C /target"
+    else
+        echo "Warning: Backup file not found for $volume_name"
+    fi
+done
+
+# Restore configuration if exists
+if [ -d "$BACKUP_DIR/config_backup" ]; then
+    cp -r "$BACKUP_DIR/config_backup/"* "${CONFIG_PATH}/"
+fi
+
+# Restore docker-compose.yml if exists
+if [ -f "$BACKUP_DIR/docker-compose.yml" ]; then
+    cp "$BACKUP_DIR/docker-compose.yml" .
+fi
+
+echo "Restore completed"
+
+# Start the containers
+bash ./start.sh
+
+echo "ELK stack restarted with restored data"