swtich from ELK to Loki!
All checks were successful
Test and Publish Templates / test-and-publish (push) Successful in 40s
All checks were successful
Test and Publish Templates / test-and-publish (push) Successful in 40s
This commit is contained in:
Your Name
@@ -1,163 +1,111 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Generate Filebeat configuration from template
|
||||
# This script creates a filebeat.yml configuration file with proper authentication
|
||||
# Generate Promtail configuration for Log Client
|
||||
|
||||
# Check required variables
|
||||
if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ]; then
|
||||
if [ -z "$LOGSERVER_HOST" ] || [ -z "$LOGSERVER_PORT" ] || [ -z "$LOKI_PASSWORD" ]; then
|
||||
echo "ERROR: Required environment variables not set"
|
||||
echo " LOGSERVER_HOST: ${LOGSERVER_HOST:-NOT SET}"
|
||||
echo " LOGSERVER_PORT: ${LOGSERVER_PORT:-NOT SET}"
|
||||
echo " LOKI_PASSWORD: ${LOKI_PASSWORD:-NOT SET}"
|
||||
echo ""
|
||||
echo "Please set these in config/service.env before running install"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine config directory - use CONFIG_PATH from dropshell or fallback
|
||||
if [ -n "$CONFIG_PATH" ]; then
|
||||
CONFIG_DIR="$CONFIG_PATH"
|
||||
elif [ -d "./config" ]; then
|
||||
CONFIG_DIR="./config"
|
||||
else
|
||||
CONFIG_DIR="."
|
||||
fi
|
||||
# Get actual hostname
|
||||
ACTUAL_HOSTNAME=${HOSTNAME_LABEL:-${HOSTNAME:-$(hostname 2>/dev/null || echo "unknown")}}
|
||||
|
||||
# Ensure config directory exists
|
||||
# Determine config directory
|
||||
CONFIG_DIR="${CONFIG_PATH:-./config}"
|
||||
mkdir -p "$CONFIG_DIR"
|
||||
|
||||
# Set defaults for variables if not set
|
||||
BULK_MAX_SIZE=${BULK_MAX_SIZE:-2048}
|
||||
WORKER_THREADS=${WORKER_THREADS:-1}
|
||||
QUEUE_SIZE=${QUEUE_SIZE:-4096}
|
||||
MAX_BACKOFF=${MAX_BACKOFF:-60s}
|
||||
# Generate promtail.yaml configuration
|
||||
cat > "$CONFIG_DIR/promtail.yaml" << EOF
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
# Get actual hostname from the host system
|
||||
ACTUAL_HOSTNAME=${HOSTNAME:-$(hostname 2>/dev/null || echo "unknown")}
|
||||
positions:
|
||||
filename: /tmp/positions/positions.yaml
|
||||
|
||||
# Generate filebeat.yml configuration with variable substitution
|
||||
(
|
||||
cat << 'TEMPLATE_EOF'
|
||||
# Filebeat Configuration for LogClient
|
||||
# Generated by generate-config.sh
|
||||
clients:
|
||||
- url: http://${LOKI_USER:-logclient}:${LOKI_PASSWORD}@${LOGSERVER_HOST}:${LOGSERVER_PORT}/loki/api/v1/push
|
||||
# Authentication via URL (HTTP Basic Auth)
|
||||
|
||||
# ======================== Docker Input Configuration =========================
|
||||
# Use Docker input to collect logs via Docker API
|
||||
filebeat.inputs:
|
||||
- type: docker
|
||||
enabled: true
|
||||
# Collect from all containers
|
||||
containers.ids:
|
||||
- '*'
|
||||
# Collect both stdout and stderr
|
||||
containers.stream: all
|
||||
# Combine partial log lines
|
||||
combine_partial: true
|
||||
# Add Docker metadata
|
||||
processors:
|
||||
- add_docker_metadata:
|
||||
host: "unix:///var/run/docker.sock"
|
||||
scrape_configs:
|
||||
# Docker container logs via Docker API
|
||||
- job_name: docker
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 5s
|
||||
relabel_configs:
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: 'container_name'
|
||||
- source_labels: ['__meta_docker_container_id']
|
||||
target_label: 'container_id'
|
||||
- source_labels: ['__meta_docker_container_image']
|
||||
target_label: 'image'
|
||||
- target_label: 'hostname'
|
||||
replacement: '${ACTUAL_HOSTNAME}'
|
||||
- target_label: 'job'
|
||||
replacement: 'docker'
|
||||
|
||||
# ======================== System Logs Configuration ==========================
|
||||
- type: log
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/log/syslog
|
||||
- /var/log/messages
|
||||
exclude_lines: ['^#']
|
||||
fields:
|
||||
log_type: syslog
|
||||
# System logs
|
||||
- job_name: syslog
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
hostname: ${ACTUAL_HOSTNAME}
|
||||
__path__: /var/log/syslog
|
||||
|
||||
- type: log
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/log/auth.log
|
||||
- /var/log/secure
|
||||
exclude_lines: ['^#']
|
||||
fields:
|
||||
log_type: auth
|
||||
- job_name: messages
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: messages
|
||||
hostname: ${ACTUAL_HOSTNAME}
|
||||
__path__: /var/log/messages
|
||||
|
||||
# ======================== Processors Configuration ===========================
|
||||
processors:
|
||||
- add_host_metadata:
|
||||
when.not.contains:
|
||||
tags: forwarded
|
||||
# Override hostname with actual host's hostname
|
||||
- add_fields:
|
||||
target: agent
|
||||
fields:
|
||||
hostname: __ACTUAL_HOSTNAME__
|
||||
- add_fields:
|
||||
target: host
|
||||
fields:
|
||||
name: __ACTUAL_HOSTNAME__
|
||||
- job_name: auth
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: auth
|
||||
hostname: ${ACTUAL_HOSTNAME}
|
||||
__path__: /var/log/auth.log
|
||||
|
||||
# ======================== Output Configuration ===============================
|
||||
output.logstash:
|
||||
hosts: ["__LOGSERVER_HOST__:__LOGSERVER_PORT__"]
|
||||
# SSL/TLS configuration
|
||||
ssl.enabled: false # Set to true when using TLS
|
||||
ssl.verification_mode: none # Set to full in production with proper certs
|
||||
# Docker container JSON logs (backup method)
|
||||
- job_name: containers
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: containers
|
||||
hostname: ${ACTUAL_HOSTNAME}
|
||||
__path__: /var/lib/docker/containers/*/*-json.log
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
output: log
|
||||
stream: stream
|
||||
time: time
|
||||
- timestamp:
|
||||
source: time
|
||||
format: RFC3339Nano
|
||||
- labels:
|
||||
stream:
|
||||
- output:
|
||||
source: output
|
||||
EOF
|
||||
|
||||
# Performance settings
|
||||
bulk_max_size: __BULK_MAX_SIZE__
|
||||
worker: __WORKER_THREADS__ # Must be >= 1
|
||||
compression_level: 3
|
||||
|
||||
# Retry configuration
|
||||
max_retries: 3
|
||||
backoff.init: 1s
|
||||
backoff.max: __MAX_BACKOFF__
|
||||
|
||||
# ======================== Global Fields =======================================
|
||||
# Add API key as a field to all events
|
||||
fields:
|
||||
api_key: "__API_KEY__"
|
||||
fields_under_root: false
|
||||
|
||||
# ======================== Queue Configuration ================================
|
||||
queue.mem:
|
||||
events: __QUEUE_SIZE__
|
||||
flush.min_events: 512
|
||||
flush.timeout: 5s
|
||||
|
||||
# ======================== Logging Configuration ==============================
|
||||
logging.level: info
|
||||
logging.to_files: true
|
||||
logging.files:
|
||||
path: /usr/share/filebeat/data/logs
|
||||
name: filebeat
|
||||
keepfiles: 3
|
||||
permissions: 0600
|
||||
|
||||
# ======================== Monitoring ==========================================
|
||||
monitoring.enabled: false
|
||||
http.enabled: true
|
||||
http.host: 0.0.0.0
|
||||
http.port: 5066
|
||||
|
||||
# ======================== File Permissions ====================================
|
||||
# Set strict permissions (disabled for Docker)
|
||||
# filebeat.config.modules.path: ${path.config}/modules.d/*.yml
|
||||
TEMPLATE_EOF
|
||||
) | sed -e "s|__LOGSERVER_HOST__|${LOGSERVER_HOST}|g" \
|
||||
-e "s|__LOGSERVER_PORT__|${LOGSERVER_PORT}|g" \
|
||||
-e "s|__API_KEY__|${API_KEY}|g" \
|
||||
-e "s|__BULK_MAX_SIZE__|${BULK_MAX_SIZE}|g" \
|
||||
-e "s|__WORKER_THREADS__|${WORKER_THREADS}|g" \
|
||||
-e "s|__QUEUE_SIZE__|${QUEUE_SIZE}|g" \
|
||||
-e "s|__MAX_BACKOFF__|${MAX_BACKOFF}|g" \
|
||||
-e "s|__ACTUAL_HOSTNAME__|${ACTUAL_HOSTNAME}|g" > "$CONFIG_DIR/filebeat.yml"
|
||||
|
||||
echo "Filebeat configuration generated at: $CONFIG_DIR/filebeat.yml"
|
||||
echo "Promtail configuration generated at: $CONFIG_DIR/promtail.yaml"
|
||||
echo "Configuration:"
|
||||
echo " LOGSERVER_HOST: ${LOGSERVER_HOST}"
|
||||
echo " LOGSERVER_PORT: ${LOGSERVER_PORT}"
|
||||
echo " API_KEY: ${API_KEY:+[SET]}"
|
||||
echo " WORKER_THREADS: ${WORKER_THREADS}"
|
||||
|
||||
# Additional warning if API_KEY is not set
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo ""
|
||||
echo "WARNING: API_KEY is not set - logs may be rejected by the server"
|
||||
echo "Get an API key from the LogServer admin using generate-api-key.sh"
|
||||
fi
|
||||
echo " HOSTNAME: ${ACTUAL_HOSTNAME}"
|
||||
Reference in New Issue
Block a user