From 7851aa810b5627fe9a0bd2c095f4a6cd9d8d1325 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Sep 2025 10:33:08 +1200 Subject: [PATCH] significant tidying to logserver --- logclient/_volumes.sh | 7 ++++ logclient/backup.sh | 32 +++++++++++++++++ logclient/config/.template_info.env | 5 ++- logclient/destroy.sh | 22 ++++++++++++ logclient/install.sh | 20 +++-------- logclient/restore.sh | 47 ++++++++++++++++++++++++ logclient/start.sh | 25 +++++++------ logserver/_volumes.sh | 9 +++++ logserver/backup.sh | 37 +++++++++++++++++++ logserver/destroy.sh | 26 ++++++++++++++ logserver/install.sh | 5 +++ logserver/restore.sh | 56 +++++++++++++++++++++++++++++ 12 files changed, 262 insertions(+), 29 deletions(-) create mode 100755 logclient/_volumes.sh create mode 100755 logclient/backup.sh create mode 100755 logclient/destroy.sh create mode 100755 logclient/restore.sh create mode 100755 logserver/_volumes.sh create mode 100755 logserver/backup.sh create mode 100755 logserver/destroy.sh create mode 100755 logserver/restore.sh diff --git a/logclient/_volumes.sh b/logclient/_volumes.sh new file mode 100755 index 0000000..32c80e5 --- /dev/null +++ b/logclient/_volumes.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Define volume items for logclient container +# These are used across backup, restore, create, and destroy operations + +get_logclient_volumes() { + echo "volume:datavolume:${DATA_VOLUME}" "volume:configvolume:${CONFIG_VOLUME}" +} \ No newline at end of file diff --git a/logclient/backup.sh b/logclient/backup.sh new file mode 100755 index 0000000..32d3939 --- /dev/null +++ b/logclient/backup.sh @@ -0,0 +1,32 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +echo "Backing up ${CONTAINER_NAME} volumes..." + +# Stop the container to ensure data consistency +bash ./stop.sh || true + +# Backup volumes +BACKUP_DIR="${CONFIG_PATH}/backups/$(date +%Y%m%d_%H%M%S)" +mkdir -p "$BACKUP_DIR" + +# Export volumes +for volume in $(get_logclient_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + echo "Backing up volume: $volume_name" + docker run --rm -v "$volume_name:/source:ro" -v "$BACKUP_DIR:/backup" alpine \ + tar -czf "/backup/${volume_name}.tar.gz" -C /source . +done + +# Backup configuration +cp -r "${CONFIG_PATH}" "$BACKUP_DIR/config_backup" + +echo "Backup completed to: $BACKUP_DIR" + +# Restart the container +bash ./start.sh + +echo "Container restarted" \ No newline at end of file diff --git a/logclient/config/.template_info.env b/logclient/config/.template_info.env index 64ec230..d82e5e0 100644 --- a/logclient/config/.template_info.env +++ b/logclient/config/.template_info.env @@ -12,6 +12,5 @@ IMAGE_REPO="beats/filebeat" IMAGE_TAG="7.17.23" # Volume definitions -CONFIG_VOLUME="${CONTAINER_NAME}_config" -DATA_VOLUME="${CONTAINER_NAME}_data" -CERTS_VOLUME="${CONTAINER_NAME}_certs" \ No newline at end of file +DATA_VOLUME=logclient_data +CONFIG_VOLUME=logclient_config \ No newline at end of file diff --git a/logclient/destroy.sh b/logclient/destroy.sh new file mode 100755 index 0000000..5243f22 --- /dev/null +++ b/logclient/destroy.sh @@ -0,0 +1,22 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +echo "WARNING: This will permanently delete all data for ${CONTAINER_NAME}" +echo "This action cannot be undone!" +echo "" + +# Since scripts must be non-interactive, only proceed if explicitly called +bash ./stop.sh || true +_remove_container "$CONTAINER_NAME" || true + +# Remove all volumes +for volume in $(get_logclient_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + echo "Removing volume: $volume_name" + docker volume rm "$volume_name" 2>/dev/null || true +done + +echo "Service and all data destroyed" \ No newline at end of file diff --git a/logclient/install.sh b/logclient/install.sh index 98c33eb..03df50e 100755 --- a/logclient/install.sh +++ b/logclient/install.sh @@ -39,22 +39,10 @@ echo "Generating Filebeat configuration..." SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" bash "$SCRIPT_DIR/scripts/generate-config.sh" || _die "Failed to generate configuration" -# Create Docker volumes -CONFIG_VOLUME="${CONFIG_VOLUME:-${CONTAINER_NAME}_config}" -DATA_VOLUME="${DATA_VOLUME:-${CONTAINER_NAME}_data}" -CERTS_VOLUME="${CERTS_VOLUME:-${CONTAINER_NAME}_certs}" - -echo "Creating Docker volumes..." -docker volume create "$CONFIG_VOLUME" >/dev/null 2>&1 || true -docker volume create "$DATA_VOLUME" >/dev/null 2>&1 || true -docker volume create "$CERTS_VOLUME" >/dev/null 2>&1 || true - -# Copy config to volume -if [ -f "${CONFIG_PATH}/filebeat.yml" ]; then - echo "Copying configuration to Docker volume..." - docker run --rm -v "${CONFIG_VOLUME}:/config" -v "${CONFIG_PATH}:/source:ro" alpine \ - cp /source/filebeat.yml /config/filebeat.yml -fi +# Create volumes using common function +source "$SCRIPT_DIR/_volumes.sh" +echo "Creating volumes..." +create_items $(get_logclient_volumes) # Start the new container bash ./start.sh || _die "Failed to start Filebeat" diff --git a/logclient/restore.sh b/logclient/restore.sh new file mode 100755 index 0000000..6dc594a --- /dev/null +++ b/logclient/restore.sh @@ -0,0 +1,47 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +if [ -z "$1" ]; then + echo "Usage: $0 " + echo "Available backups:" + ls -la "${CONFIG_PATH}/backups/" 2>/dev/null || echo "No backups found" + exit 1 +fi + +BACKUP_DIR="$1" + +if [ ! -d "$BACKUP_DIR" ]; then + _die "Backup directory not found: $BACKUP_DIR" +fi + +echo "Restoring from backup: $BACKUP_DIR" + +# Stop the container +bash ./stop.sh || true + +# Restore volumes +for volume in $(get_logclient_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + backup_file="$BACKUP_DIR/${volume_name}.tar.gz" + + if [ -f "$backup_file" ]; then + echo "Restoring volume: $volume_name" + docker run --rm -v "$volume_name:/target" -v "$BACKUP_DIR:/backup:ro" alpine \ + sh -c "rm -rf /target/* && tar -xzf /backup/${volume_name}.tar.gz -C /target" + fi +done + +# Restore configuration if exists +if [ -d "$BACKUP_DIR/config_backup" ]; then + cp -r "$BACKUP_DIR/config_backup/"* "${CONFIG_PATH}/" +fi + +echo "Restore completed" + +# Start the container +bash ./start.sh + +echo "Container restarted" \ No newline at end of file diff --git a/logclient/start.sh b/logclient/start.sh index 126d642..57cc908 100755 --- a/logclient/start.sh +++ b/logclient/start.sh @@ -2,21 +2,20 @@ source "${AGENT_PATH}/common.sh" _check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" -# Define volume names if not set -CONFIG_VOLUME="${CONFIG_VOLUME:-${CONTAINER_NAME}_config}" -DATA_VOLUME="${DATA_VOLUME:-${CONTAINER_NAME}_data}" -CERTS_VOLUME="${CERTS_VOLUME:-${CONTAINER_NAME}_certs}" +# Check that config file exists +if [ ! -f "${CONFIG_PATH}/filebeat.yml" ]; then + _die "filebeat.yml not found in ${CONFIG_PATH}/filebeat.yml" +fi # Create Docker command -cmd="docker run -d \ +DOCKER_RUN_CMD="docker run -d \ --name $CONTAINER_NAME \ --restart unless-stopped \ --user root \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ -v /var/log:/var/log:ro \ - -v ${CONFIG_VOLUME}:/usr/share/filebeat/config:ro \ + -v ${CONFIG_PATH}:/usr/share/filebeat/config:ro \ -v ${DATA_VOLUME}:/usr/share/filebeat/data \ - -v ${CERTS_VOLUME}:/usr/share/filebeat/certs:ro \ -e LOGSERVER_HOST=${LOGSERVER_HOST} \ -e LOGSERVER_PORT=${LOGSERVER_PORT} \ -e API_KEY=${API_KEY} \ @@ -24,7 +23,13 @@ cmd="docker run -d \ filebeat -e -strict.perms=false \ -c /usr/share/filebeat/config/filebeat.yml" -# Start the container -eval "$cmd" || _die "Failed to start Filebeat container" +if ! _create_and_start_container "$DOCKER_RUN_CMD" "$CONTAINER_NAME"; then + _die "Failed to start container ${CONTAINER_NAME}" +fi -echo "Filebeat started successfully" \ No newline at end of file +# Check if the container is running +if ! _is_container_running "$CONTAINER_NAME"; then + _die "Container ${CONTAINER_NAME} is not running" +fi + +echo "Container ${CONTAINER_NAME} started" \ No newline at end of file diff --git a/logserver/_volumes.sh b/logserver/_volumes.sh new file mode 100755 index 0000000..ee9801b --- /dev/null +++ b/logserver/_volumes.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Define volume items for logserver container +# These are used across backup, restore, create, and destroy operations + +get_logserver_volumes() { + echo "volume:elasticsearch_data:${CONTAINER_NAME}_elasticsearch_data" \ + "volume:logstash_data:${CONTAINER_NAME}_logstash_data" \ + "volume:kibana_data:${CONTAINER_NAME}_kibana_data" +} \ No newline at end of file diff --git a/logserver/backup.sh b/logserver/backup.sh new file mode 100755 index 0000000..5ffca67 --- /dev/null +++ b/logserver/backup.sh @@ -0,0 +1,37 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +echo "Backing up ${CONTAINER_NAME} ELK stack volumes..." +echo "Note: This may take a while for large log databases" + +# Stop the containers to ensure data consistency +bash ./stop.sh || true + +# Backup volumes +BACKUP_DIR="${CONFIG_PATH}/backups/$(date +%Y%m%d_%H%M%S)" +mkdir -p "$BACKUP_DIR" + +# Export volumes +for volume in $(get_logserver_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + echo "Backing up volume: $volume_name" + docker run --rm -v "$volume_name:/source:ro" -v "$BACKUP_DIR:/backup" alpine \ + tar -czf "/backup/${volume_name}.tar.gz" -C /source . +done + +# Backup configuration +cp -r "${CONFIG_PATH}" "$BACKUP_DIR/config_backup" + +# Backup docker-compose.yml +cp docker-compose.yml "$BACKUP_DIR/" 2>/dev/null || true + +echo "Backup completed to: $BACKUP_DIR" +echo "Size: $(du -sh $BACKUP_DIR | cut -f1)" + +# Restart the containers +bash ./start.sh + +echo "ELK stack restarted" \ No newline at end of file diff --git a/logserver/destroy.sh b/logserver/destroy.sh new file mode 100755 index 0000000..0906253 --- /dev/null +++ b/logserver/destroy.sh @@ -0,0 +1,26 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +echo "WARNING: This will PERMANENTLY DELETE all ELK stack data!" +echo "This includes all logs, dashboards, and configurations" +echo "This action cannot be undone!" +echo "" + +# Since scripts must be non-interactive, only proceed if explicitly called +# Stop all containers +docker compose down || true + +# Remove all volumes +for volume in $(get_logserver_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + echo "Removing volume: $volume_name" + docker volume rm "$volume_name" 2>/dev/null || true +done + +# Also remove the Docker network if it exists +docker network rm "${CONTAINER_NAME}_elk" 2>/dev/null || true + +echo "ELK stack and all data destroyed" \ No newline at end of file diff --git a/logserver/install.sh b/logserver/install.sh index 48060dc..df0be9f 100755 --- a/logserver/install.sh +++ b/logserver/install.sh @@ -31,6 +31,11 @@ docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} || _die docker pull docker.elastic.co/logstash/logstash:${LS_VERSION} || _die "Failed to pull Logstash" docker pull docker.elastic.co/kibana/kibana:${KIBANA_VERSION} || _die "Failed to pull Kibana" +# Create volumes using common function +source "$SCRIPT_DIR/_volumes.sh" +echo "Creating volumes..." +create_items $(get_logserver_volumes) + # Ensure config directory exists mkdir -p "${CONFIG_PATH}" diff --git a/logserver/restore.sh b/logserver/restore.sh new file mode 100755 index 0000000..b734068 --- /dev/null +++ b/logserver/restore.sh @@ -0,0 +1,56 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +source "$(dirname "${BASH_SOURCE[0]}")/_volumes.sh" + +_check_required_env_vars "CONTAINER_NAME" + +if [ -z "$1" ]; then + echo "Usage: $0 " + echo "Available backups:" + ls -la "${CONFIG_PATH}/backups/" 2>/dev/null || echo "No backups found" + exit 1 +fi + +BACKUP_DIR="$1" + +if [ ! -d "$BACKUP_DIR" ]; then + _die "Backup directory not found: $BACKUP_DIR" +fi + +echo "Restoring ELK stack from backup: $BACKUP_DIR" +echo "WARNING: This will overwrite all current data!" + +# Stop the containers +bash ./stop.sh || true + +# Restore volumes +for volume in $(get_logserver_volumes); do + volume_name=$(echo $volume | cut -d: -f3) + backup_file="$BACKUP_DIR/${volume_name}.tar.gz" + + if [ -f "$backup_file" ]; then + echo "Restoring volume: $volume_name" + # Clear existing data and restore + docker run --rm -v "$volume_name:/target" -v "$BACKUP_DIR:/backup:ro" alpine \ + sh -c "rm -rf /target/* && tar -xzf /backup/${volume_name}.tar.gz -C /target" + else + echo "Warning: Backup file not found for $volume_name" + fi +done + +# Restore configuration if exists +if [ -d "$BACKUP_DIR/config_backup" ]; then + cp -r "$BACKUP_DIR/config_backup/"* "${CONFIG_PATH}/" +fi + +# Restore docker-compose.yml if exists +if [ -f "$BACKUP_DIR/docker-compose.yml" ]; then + cp "$BACKUP_DIR/docker-compose.yml" . +fi + +echo "Restore completed" + +# Start the containers +bash ./start.sh + +echo "ELK stack restarted with restored data" \ No newline at end of file