From e7ce7ec0c5f8cceb4cc3189160e88418db8aec04 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Sep 2025 11:05:54 +1200 Subject: [PATCH] fixes fixes fixes but more needed --- logclient/scripts/generate-config.sh | 58 +++++++++++++++++++++------- logserver/config/logstash.yml | 28 ++++++++++++++ logserver/docker-compose.yml | 2 +- logserver/install.sh | 16 ++++++-- 4 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 logserver/config/logstash.yml diff --git a/logclient/scripts/generate-config.sh b/logclient/scripts/generate-config.sh index 92590d6..cab766f 100755 --- a/logclient/scripts/generate-config.sh +++ b/logclient/scripts/generate-config.sh @@ -3,6 +3,16 @@ # Generate Filebeat configuration from template # This script creates a filebeat.yml configuration file with proper authentication +# Check required variables +if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ]; then + echo "ERROR: Required environment variables not set" + echo " LOGSERVER_HOST: ${LOGSERVER_HOST:-NOT SET}" + echo " LOGSERVER_PORT: ${LOGSERVER_PORT:-NOT SET}" + echo "" + echo "Please set these in config/service.env before running install" + exit 1 +fi + # Determine config directory - use CONFIG_PATH from dropshell or fallback if [ -n "$CONFIG_PATH" ]; then CONFIG_DIR="$CONFIG_PATH" @@ -15,8 +25,15 @@ fi # Ensure config directory exists mkdir -p "$CONFIG_DIR" -# Generate filebeat.yml configuration -cat > "$CONFIG_DIR/filebeat.yml" << EOF +# Set defaults for variables if not set +BULK_MAX_SIZE=${BULK_MAX_SIZE:-2048} +WORKER_THREADS=${WORKER_THREADS:-1} +QUEUE_SIZE=${QUEUE_SIZE:-4096} +MAX_BACKOFF=${MAX_BACKOFF:-60s} + +# Generate filebeat.yml configuration with variable substitution +( +cat << 'TEMPLATE_EOF' # Filebeat Configuration for LogClient # Generated by generate-config.sh @@ -64,30 +81,30 @@ processors: # ======================== Output Configuration =============================== output.logstash: - hosts: ["${LOGSERVER_HOST}:${LOGSERVER_PORT}"] + hosts: ["__LOGSERVER_HOST__:__LOGSERVER_PORT__"] # SSL/TLS configuration ssl.enabled: false # Set to true when using TLS ssl.verification_mode: none # Set to full in production with proper certs # Performance settings - bulk_max_size: ${BULK_MAX_SIZE:-2048} - worker: ${WORKER_THREADS:-1} # Must be >= 1 + bulk_max_size: __BULK_MAX_SIZE__ + worker: __WORKER_THREADS__ # Must be >= 1 compression_level: 3 # Retry configuration max_retries: 3 backoff.init: 1s - backoff.max: ${MAX_BACKOFF:-60s} + backoff.max: __MAX_BACKOFF__ # ======================== Global Fields ======================================= # Add API key as a field to all events fields: - api_key: "${API_KEY}" + api_key: "__API_KEY__" fields_under_root: false # ======================== Queue Configuration ================================ queue.mem: - events: ${QUEUE_SIZE:-4096} + events: __QUEUE_SIZE__ flush.min_events: 512 flush.timeout: 5s @@ -109,14 +126,25 @@ http.port: 5066 # ======================== File Permissions ==================================== # Set strict permissions (disabled for Docker) # filebeat.config.modules.path: ${path.config}/modules.d/*.yml -EOF +TEMPLATE_EOF +) | sed -e "s|__LOGSERVER_HOST__|${LOGSERVER_HOST}|g" \ + -e "s|__LOGSERVER_PORT__|${LOGSERVER_PORT}|g" \ + -e "s|__API_KEY__|${API_KEY}|g" \ + -e "s|__BULK_MAX_SIZE__|${BULK_MAX_SIZE}|g" \ + -e "s|__WORKER_THREADS__|${WORKER_THREADS}|g" \ + -e "s|__QUEUE_SIZE__|${QUEUE_SIZE}|g" \ + -e "s|__MAX_BACKOFF__|${MAX_BACKOFF}|g" > "$CONFIG_DIR/filebeat.yml" echo "Filebeat configuration generated at: $CONFIG_DIR/filebeat.yml" +echo "Configuration:" +echo " LOGSERVER_HOST: ${LOGSERVER_HOST}" +echo " LOGSERVER_PORT: ${LOGSERVER_PORT}" +echo " API_KEY: ${API_KEY:+[SET]}" +echo " WORKER_THREADS: ${WORKER_THREADS}" -# Validate that required environment variables are set -if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ] || [ -z "$API_KEY" ]; then - echo "WARNING: Required environment variables not set" - echo " LOGSERVER_HOST: ${LOGSERVER_HOST:-NOT SET}" - echo " LOGSERVER_PORT: ${LOGSERVER_PORT:-NOT SET}" - echo " API_KEY: ${API_KEY:+SET}" +# Additional warning if API_KEY is not set +if [ -z "$API_KEY" ]; then + echo "" + echo "WARNING: API_KEY is not set - logs may be rejected by the server" + echo "Get an API key from the LogServer admin using generate-api-key.sh" fi \ No newline at end of file diff --git a/logserver/config/logstash.yml b/logserver/config/logstash.yml new file mode 100644 index 0000000..1aed618 --- /dev/null +++ b/logserver/config/logstash.yml @@ -0,0 +1,28 @@ +# Logstash Configuration Settings +# This file contains Logstash settings (not pipeline configuration) + +# Node name +node.name: "${CONTAINER_NAME}_logstash" + +# Pipeline settings +pipeline.workers: 2 +pipeline.batch.size: 125 +pipeline.batch.delay: 50 + +# HTTP API settings +http.host: "0.0.0.0" +http.port: 9600 + +# Monitoring +monitoring.enabled: false + +# Queue settings +queue.type: memory +queue.max_bytes: 1gb + +# Path settings are handled by Docker volumes +# path.data: /usr/share/logstash/data +# path.logs: /usr/share/logstash/logs + +# Log level +log.level: info \ No newline at end of file diff --git a/logserver/docker-compose.yml b/logserver/docker-compose.yml index 1be7eff..f99d19d 100644 --- a/logserver/docker-compose.yml +++ b/logserver/docker-compose.yml @@ -34,7 +34,7 @@ services: - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-${KIBANA_PASSWORD:-changeme}} command: logstash -f /usr/share/logstash/config/logstash.conf volumes: - - ${CONFIG_PATH}:/usr/share/logstash/config:ro + - ${CONFIG_PATH}:/usr/share/logstash/config - logstash_data:/usr/share/logstash/data ports: - "${LOGSTASH_BEATS_PORT:-5044}:5044" diff --git a/logserver/install.sh b/logserver/install.sh index e233ed2..6806c06 100755 --- a/logserver/install.sh +++ b/logserver/install.sh @@ -55,17 +55,27 @@ if [ ! -f "${CONFIG_PATH}/api-keys.yml" ]; then echo "api_keys:" > "${CONFIG_PATH}/api-keys.yml" fi -# Copy Logstash configuration if it doesn't exist +# Copy Logstash configurations if they don't exist +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if [ ! -f "${CONFIG_PATH}/logstash.conf" ]; then - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -f "$SCRIPT_DIR/config/logstash.conf" ]; then cp "$SCRIPT_DIR/config/logstash.conf" "${CONFIG_PATH}/logstash.conf" - echo "Copied Logstash configuration to ${CONFIG_PATH}" + echo "Copied Logstash pipeline configuration to ${CONFIG_PATH}" else echo "WARNING: logstash.conf not found in template" fi fi +if [ ! -f "${CONFIG_PATH}/logstash.yml" ]; then + if [ -f "$SCRIPT_DIR/config/logstash.yml" ]; then + cp "$SCRIPT_DIR/config/logstash.yml" "${CONFIG_PATH}/logstash.yml" + echo "Copied Logstash settings to ${CONFIG_PATH}" + else + echo "WARNING: logstash.yml not found in template" + fi +fi + # Start the ELK stack echo "Starting ELK stack..." docker compose up -d --build || _die "Failed to start ELK stack"