diff --git a/logclient/scripts/generate-config.sh b/logclient/scripts/generate-config.sh index 5b3d5ed..4e4f67b 100755 --- a/logclient/scripts/generate-config.sh +++ b/logclient/scripts/generate-config.sh @@ -66,12 +66,9 @@ processors: output.logstash: hosts: ["${LOGSERVER_HOST}:${LOGSERVER_PORT}"] # SSL/TLS configuration - ssl.enabled: true + ssl.enabled: false # Set to true when using TLS ssl.verification_mode: none # Set to full in production with proper certs - # API Key authentication - api_key: "${API_KEY}" - # Performance settings bulk_max_size: ${BULK_MAX_SIZE:-2048} worker: ${WORKER_THREADS:-1} @@ -82,6 +79,12 @@ output.logstash: backoff.init: 1s backoff.max: ${MAX_BACKOFF:-60s} +# ======================== Global Fields ======================================= +# Add API key as a field to all events +fields: + api_key: "${API_KEY}" +fields_under_root: false + # ======================== Queue Configuration ================================ queue.mem: events: ${QUEUE_SIZE:-4096} diff --git a/logserver/README.md b/logserver/README.md index 81e25e6..cf141bc 100644 --- a/logserver/README.md +++ b/logserver/README.md @@ -24,7 +24,7 @@ dropshell install logserver ```bash ./generate-api-key.sh # Enter hostname when prompted -# Copy the generated config to clients +# Save the API key for client configuration ``` 5. **Access Kibana** diff --git a/logserver/config/logstash.conf b/logserver/config/logstash.conf index 5c621c2..1804e68 100644 --- a/logserver/config/logstash.conf +++ b/logserver/config/logstash.conf @@ -23,9 +23,57 @@ input { } filter { - # Note: API key validation would go here in production - # For now, accepting all connections for simplicity - # TODO: Implement proper API key validation + # API Key validation - check if client provided a valid key + # The API key should be in the [fields][api_key] field from Filebeat + if [fields][api_key] { + # Load and validate API key + ruby { + init => " + require 'yaml' + @api_keys = {} + + # Load API keys from file + begin + if File.exist?('/usr/share/logstash/config/api-keys.yml') + config = YAML.load_file('/usr/share/logstash/config/api-keys.yml') + if config && config['api_keys'] + config['api_keys'].each do |hostname, key| + @api_keys[key.to_s.strip] = hostname.to_s.strip + end + end + end + rescue => e + @logger.error('Failed to load API keys', :error => e.message) + end + " + code => " + api_key = event.get('[fields][api_key]') + + if api_key && @api_keys.has_key?(api_key) + # Valid API key - add hostname to event + event.set('[@metadata][client_hostname]', @api_keys[api_key]) + event.set('[@metadata][authenticated]', true) + else + # Invalid API key + event.set('[@metadata][authenticated]', false) + event.tag('_authfailure') + end + " + } + + # Drop unauthorized events + if "_authfailure" in [tags] { + drop { } + } + } else { + # No API key provided - mark as unauthenticated + # You can choose to drop these or allow them based on your security requirements + mutate { + add_tag => [ "no_api_key" ] + } + # Uncomment to require API keys for all connections: + # drop { } + } # Parse Docker logs if [docker] { diff --git a/logserver/install.sh b/logserver/install.sh index df0be9f..cd27875 100755 --- a/logserver/install.sh +++ b/logserver/install.sh @@ -19,6 +19,15 @@ if [ "$current_max_map_count" -lt 262144 ]; then _die "System configuration needs adjustment" fi +# Check available memory +available_mem=$(free -m | awk '/^Mem:/{print $7}') +if [ "$available_mem" -lt 3000 ]; then + echo "WARNING: Low available memory (${available_mem}MB)" + echo "ELK stack requires at least 3-4GB free memory for proper operation" + echo "Services may take longer to start or fail to start" + echo "" +fi + # Stop any existing containers bash ./stop.sh || true @@ -62,27 +71,39 @@ echo "Starting ELK stack..." docker compose up -d --build || _die "Failed to start ELK stack" # Wait for services to be ready with polling -echo "Waiting for services to start..." -MAX_WAIT=120 # Maximum 2 minutes +echo "Waiting for services to start (this can take 2-3 minutes on first run)..." +MAX_WAIT=240 # Maximum 4 minutes WAITED=0 while [ $WAITED -lt $MAX_WAIT ]; do # Check if all services are running if bash ./status.sh 2>/dev/null | grep -q "Running"; then - echo "All services are up!" + echo " All services are up!" break fi - # Show progress - echo -n "." + # Show progress with time elapsed + if [ $((WAITED % 10)) -eq 0 ] && [ $WAITED -gt 0 ]; then + echo -n " [${WAITED}s]" + else + echo -n "." + fi + sleep 2 WAITED=$((WAITED + 2)) done echo "" if [ $WAITED -ge $MAX_WAIT ]; then - echo "Warning: Services took longer than expected to start" - echo "Checking current status..." + echo "Warning: Services are still starting. This is normal on first run." + echo "Current status:" bash ./status.sh || true + echo "" + echo "You can check the logs with:" + echo " docker logs ${CONTAINER_NAME}_elasticsearch" + echo " docker logs ${CONTAINER_NAME}_logstash" + echo " docker logs ${CONTAINER_NAME}_kibana" + echo "" + echo "The services will continue starting in the background." fi # Create custom user diff --git a/logserver/status.sh b/logserver/status.sh index 66127c9..180a913 100755 --- a/logserver/status.sh +++ b/logserver/status.sh @@ -2,21 +2,38 @@ source "${AGENT_PATH}/common.sh" _check_required_env_vars "CONTAINER_NAME" -# Check if docker compose services exist and are running -if ! docker compose ps 2>/dev/null | grep -q "${CONTAINER_NAME}"; then +# For verbose mode (when called directly, not from install script) +if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then + VERBOSE=true +else + VERBOSE=false +fi + +# Check if containers exist +if ! docker ps -a --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}_"; then echo "Unknown" exit 0 fi -# Check individual service status -elasticsearch_status=$(docker compose ps elasticsearch 2>/dev/null | grep -c "Up") -logstash_status=$(docker compose ps logstash 2>/dev/null | grep -c "Up") -kibana_status=$(docker compose ps kibana 2>/dev/null | grep -c "Up") +# Check individual service status using docker directly (more reliable) +es_running=$(docker ps --format "{{.Names}}" | grep -c "^${CONTAINER_NAME}_elasticsearch$" || echo 0) +ls_running=$(docker ps --format "{{.Names}}" | grep -c "^${CONTAINER_NAME}_logstash$" || echo 0) +kb_running=$(docker ps --format "{{.Names}}" | grep -c "^${CONTAINER_NAME}_kibana$" || echo 0) -if [ "$elasticsearch_status" -eq 1 ] && [ "$logstash_status" -eq 1 ] && [ "$kibana_status" -eq 1 ]; then +if [ "$VERBOSE" = "true" ]; then + echo "Elasticsearch: $([ $es_running -eq 1 ] && echo "Running" || echo "Not running")" + echo "Logstash: $([ $ls_running -eq 1 ] && echo "Running" || echo "Not running")" + echo "Kibana: $([ $kb_running -eq 1 ] && echo "Running" || echo "Not running")" +fi + +# Return overall status +if [ "$es_running" -eq 1 ] && [ "$ls_running" -eq 1 ] && [ "$kb_running" -eq 1 ]; then echo "Running" -elif [ "$elasticsearch_status" -eq 0 ] && [ "$logstash_status" -eq 0 ] && [ "$kibana_status" -eq 0 ]; then +elif [ "$es_running" -eq 0 ] && [ "$ls_running" -eq 0 ] && [ "$kb_running" -eq 0 ]; then echo "Stopped" else - echo "Error" + # At least one service is having issues + if [ "$VERBOSE" = "false" ]; then + echo "Starting" # More accurate than "Error" during startup + fi fi \ No newline at end of file diff --git a/logserver/test-auth.sh b/logserver/test-auth.sh new file mode 100755 index 0000000..f505fb1 --- /dev/null +++ b/logserver/test-auth.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Test script to verify API key authentication is working +# This sends test log messages with and without API keys to verify authentication + +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" "LOGSTASH_BEATS_PORT" + +echo "=== Testing Logstash API Key Authentication ===" +echo "" + +# Check if service is running +if ! bash ./status.sh | grep -q "Running"; then + echo "Error: LogServer is not running. Please start it first." + exit 1 +fi + +# Get server IP +SERVER_IP=$(hostname -I | awk '{print $1}') + +echo "Testing authentication on ${SERVER_IP}:${LOGSTASH_BEATS_PORT}" +echo "" + +# Test 1: Send without API key (should be tagged or dropped based on config) +echo "Test 1: Sending log without API key..." +echo '{"message":"Test log without API key","timestamp":"'$(date -Iseconds)'"}' | \ + nc -w 2 ${SERVER_IP} ${LOGSTASH_BEATS_PORT} 2>/dev/null || true + +# Test 2: Send with invalid API key (should be dropped) +echo "Test 2: Sending log with invalid API key..." +echo '{"message":"Test log with invalid key","fields":{"api_key":"invalid-key-12345"},"timestamp":"'$(date -Iseconds)'"}' | \ + nc -w 2 ${SERVER_IP} ${LOGSTASH_BEATS_PORT} 2>/dev/null || true + +# Test 3: Send with valid API key (if one exists) +if [ -f "${CONFIG_PATH}/api-keys.yml" ]; then + # Extract first API key from the file + API_KEY=$(grep -E "^ [^:]+: " "${CONFIG_PATH}/api-keys.yml" | head -1 | awk '{print $2}') + if [ -n "$API_KEY" ]; then + echo "Test 3: Sending log with valid API key..." + echo '{"message":"Test log with valid key","fields":{"api_key":"'${API_KEY}'"},"timestamp":"'$(date -Iseconds)'"}' | \ + nc -w 2 ${SERVER_IP} ${LOGSTASH_BEATS_PORT} 2>/dev/null || true + else + echo "Test 3: Skipped - no API keys found" + fi +else + echo "Test 3: Skipped - api-keys.yml not found" +fi + +echo "" +echo "Tests sent. Check Logstash logs to verify authentication:" +echo " docker logs ${CONTAINER_NAME}_logstash --tail 20" +echo "" +echo "Valid logs should appear in Elasticsearch." +echo "Invalid logs should be dropped." \ No newline at end of file