fixes fixes fixes but more needed

logclient/scripts/generate-config.sh

@@ -3,6 +3,16 @@
 # Generate Filebeat configuration from template
 # This script creates a filebeat.yml configuration file with proper authentication
 
+# Check required variables
+if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ]; then
+    echo "ERROR: Required environment variables not set"
+    echo "  LOGSERVER_HOST: ${LOGSERVER_HOST:-NOT SET}"
+    echo "  LOGSERVER_PORT: ${LOGSERVER_PORT:-NOT SET}"
+    echo ""
+    echo "Please set these in config/service.env before running install"
+    exit 1
+fi
+
 # Determine config directory - use CONFIG_PATH from dropshell or fallback
 if [ -n "$CONFIG_PATH" ]; then
     CONFIG_DIR="$CONFIG_PATH"
@@ -15,8 +25,15 @@ fi
 # Ensure config directory exists
 mkdir -p "$CONFIG_DIR"
 
-# Generate filebeat.yml configuration
+# Set defaults for variables if not set
-cat > "$CONFIG_DIR/filebeat.yml" << EOF
+BULK_MAX_SIZE=${BULK_MAX_SIZE:-2048}
+WORKER_THREADS=${WORKER_THREADS:-1}
+QUEUE_SIZE=${QUEUE_SIZE:-4096}
+MAX_BACKOFF=${MAX_BACKOFF:-60s}
+
+# Generate filebeat.yml configuration with variable substitution
+(
+cat << 'TEMPLATE_EOF'
 # Filebeat Configuration for LogClient
 # Generated by generate-config.sh
 
@@ -64,30 +81,30 @@ processors:
 
 # ======================== Output Configuration ===============================
 output.logstash:
-  hosts: ["${LOGSERVER_HOST}:${LOGSERVER_PORT}"]
+  hosts: ["__LOGSERVER_HOST__:__LOGSERVER_PORT__"]
   # SSL/TLS configuration
   ssl.enabled: false  # Set to true when using TLS
   ssl.verification_mode: none  # Set to full in production with proper certs
 
   # Performance settings
-  bulk_max_size: ${BULK_MAX_SIZE:-2048}
+  bulk_max_size: __BULK_MAX_SIZE__
-  worker: ${WORKER_THREADS:-1}  # Must be >= 1
+  worker: __WORKER_THREADS__  # Must be >= 1
   compression_level: 3
 
   # Retry configuration
   max_retries: 3
   backoff.init: 1s
-  backoff.max: ${MAX_BACKOFF:-60s}
+  backoff.max: __MAX_BACKOFF__
 
 # ======================== Global Fields =======================================
 # Add API key as a field to all events
 fields:
-  api_key: "${API_KEY}"
+  api_key: "__API_KEY__"
 fields_under_root: false
 
 # ======================== Queue Configuration ================================
 queue.mem:
-  events: ${QUEUE_SIZE:-4096}
+  events: __QUEUE_SIZE__
   flush.min_events: 512
   flush.timeout: 5s
 
@@ -109,14 +126,25 @@ http.port: 5066
 # ======================== File Permissions ====================================
 # Set strict permissions (disabled for Docker)
 # filebeat.config.modules.path: ${path.config}/modules.d/*.yml
-EOF
+TEMPLATE_EOF
+) | sed -e "s|__LOGSERVER_HOST__|${LOGSERVER_HOST}|g" \
+        -e "s|__LOGSERVER_PORT__|${LOGSERVER_PORT}|g" \
+        -e "s|__API_KEY__|${API_KEY}|g" \
+        -e "s|__BULK_MAX_SIZE__|${BULK_MAX_SIZE}|g" \
+        -e "s|__WORKER_THREADS__|${WORKER_THREADS}|g" \
+        -e "s|__QUEUE_SIZE__|${QUEUE_SIZE}|g" \
+        -e "s|__MAX_BACKOFF__|${MAX_BACKOFF}|g" > "$CONFIG_DIR/filebeat.yml"
 
 echo "Filebeat configuration generated at: $CONFIG_DIR/filebeat.yml"
+echo "Configuration:"
+echo "  LOGSERVER_HOST: ${LOGSERVER_HOST}"
+echo "  LOGSERVER_PORT: ${LOGSERVER_PORT}"
+echo "  API_KEY: ${API_KEY:+[SET]}"
+echo "  WORKER_THREADS: ${WORKER_THREADS}"
 
-# Validate that required environment variables are set
+# Additional warning if API_KEY is not set
-if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ] || [ -z "$API_KEY" ]; then
+if [ -z "$API_KEY" ]; then
-    echo "WARNING: Required environment variables not set"
+    echo ""
-    echo "  LOGSERVER_HOST: ${LOGSERVER_HOST:-NOT SET}"
+    echo "WARNING: API_KEY is not set - logs may be rejected by the server"
-    echo "  LOGSERVER_PORT: ${LOGSERVER_PORT:-NOT SET}"
+    echo "Get an API key from the LogServer admin using generate-api-key.sh"
-    echo "  API_KEY: ${API_KEY:+SET}"
 fi

logserver/config/logstash.yml

@@ -0,0 +1,28 @@
+# Logstash Configuration Settings
+# This file contains Logstash settings (not pipeline configuration)
+
+# Node name
+node.name: "${CONTAINER_NAME}_logstash"
+
+# Pipeline settings
+pipeline.workers: 2
+pipeline.batch.size: 125
+pipeline.batch.delay: 50
+
+# HTTP API settings
+http.host: "0.0.0.0"
+http.port: 9600
+
+# Monitoring
+monitoring.enabled: false
+
+# Queue settings
+queue.type: memory
+queue.max_bytes: 1gb
+
+# Path settings are handled by Docker volumes
+# path.data: /usr/share/logstash/data
+# path.logs: /usr/share/logstash/logs
+
+# Log level
+log.level: info

logserver/docker-compose.yml

@@ -34,7 +34,7 @@ services:
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-${KIBANA_PASSWORD:-changeme}}
     command: logstash -f /usr/share/logstash/config/logstash.conf
     volumes:
-      - ${CONFIG_PATH}:/usr/share/logstash/config:ro
+      - ${CONFIG_PATH}:/usr/share/logstash/config
       - logstash_data:/usr/share/logstash/data
     ports:
       - "${LOGSTASH_BEATS_PORT:-5044}:5044"

logserver/install.sh

@@ -55,17 +55,27 @@ if [ ! -f "${CONFIG_PATH}/api-keys.yml" ]; then
     echo "api_keys:" > "${CONFIG_PATH}/api-keys.yml"
 fi
 
-# Copy Logstash configuration if it doesn't exist
+# Copy Logstash configurations if they don't exist
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 if [ ! -f "${CONFIG_PATH}/logstash.conf" ]; then
-    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
     if [ -f "$SCRIPT_DIR/config/logstash.conf" ]; then
         cp "$SCRIPT_DIR/config/logstash.conf" "${CONFIG_PATH}/logstash.conf"
-        echo "Copied Logstash configuration to ${CONFIG_PATH}"
+        echo "Copied Logstash pipeline configuration to ${CONFIG_PATH}"
     else
         echo "WARNING: logstash.conf not found in template"
     fi
 fi
 
+if [ ! -f "${CONFIG_PATH}/logstash.yml" ]; then
+    if [ -f "$SCRIPT_DIR/config/logstash.yml" ]; then
+        cp "$SCRIPT_DIR/config/logstash.yml" "${CONFIG_PATH}/logstash.yml"
+        echo "Copied Logstash settings to ${CONFIG_PATH}"
+    else
+        echo "WARNING: logstash.yml not found in template"
+    fi
+fi
+
 # Start the ELK stack
 echo "Starting ELK stack..."
 docker compose up -d --build || _die "Failed to start ELK stack"