From 47a51ec1768cbc990060c49e13d120e5f3e1594d Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 20 Sep 2025 09:04:29 +1200 Subject: [PATCH] new logging systems --- logclient/README.md | 389 ++++++++++++++++++++++++++++ logclient/TODO.md | 294 +++++++++++++++++++++ logclient/config/.template_info.env | 17 ++ logclient/config/service.env | 44 ++++ logclient/install.sh | 62 +++++ logclient/start.sh | 25 ++ logclient/status.sh | 31 +++ logclient/stop.sh | 7 + logclient/uninstall.sh | 17 ++ logserver/README.md | 279 ++++++++++++++++++++ logserver/TODO.md | 244 +++++++++++++++++ logserver/config/.template_info.env | 17 ++ logserver/config/service.env | 46 ++++ logserver/install.sh | 62 +++++ logserver/start.sh | 17 ++ logserver/status.sh | 22 ++ logserver/stop.sh | 8 + logserver/uninstall.sh | 16 ++ 18 files changed, 1597 insertions(+) create mode 100644 logclient/README.md create mode 100644 logclient/TODO.md create mode 100644 logclient/config/.template_info.env create mode 100644 logclient/config/service.env create mode 100755 logclient/install.sh create mode 100755 logclient/start.sh create mode 100755 logclient/status.sh create mode 100755 logclient/stop.sh create mode 100755 logclient/uninstall.sh create mode 100644 logserver/README.md create mode 100644 logserver/TODO.md create mode 100644 logserver/config/.template_info.env create mode 100644 logserver/config/service.env create mode 100755 logserver/install.sh create mode 100755 logserver/start.sh create mode 100755 logserver/status.sh create mode 100755 logserver/stop.sh create mode 100755 logserver/uninstall.sh diff --git a/logclient/README.md b/logclient/README.md new file mode 100644 index 0000000..44aa748 --- /dev/null +++ b/logclient/README.md @@ -0,0 +1,389 @@ +# Dropshell LogClient Template + +An auto-configuring Filebeat agent that collects Docker container logs via the Docker API and system logs, shipping them to a centralized logging server with minimal configuration. + +## Overview + +This template deploys a lightweight Filebeat agent that: +- Uses Docker API to collect logs from all containers (regardless of logging driver) +- Allows containers to use any Docker logging driver (json-file, local, journald, etc.) +- Collects system logs (syslog, auth logs, kernel logs) +- Ships logs to a centralized ELK stack (logserver) +- Requires minimal configuration - just the server address +- Handles connection failures with local buffering +- Auto-reconnects when the server becomes available + +## Features + +### Docker API Log Collection +- **Direct API access**: Reads logs via Docker API, not from files +- **Driver independent**: Works with any Docker logging driver (local, json-file, journald) +- **Automatic discovery**: Finds all running containers dynamically +- **Container metadata**: Enriches logs with container names, images, labels +- **Real-time streaming**: Gets logs as they're generated +- **Multi-line handling**: Properly handles stack traces and multi-line logs +- **JSON parsing**: Automatically parses JSON-formatted logs +- **Label-based filtering**: Can include/exclude containers based on labels + +### System Log Collection +- **/var/log/syslog** or **/var/log/messages**: System events +- **/var/log/auth.log** or **/var/log/secure**: Authentication events +- **/var/log/kern.log**: Kernel messages +- **journald**: SystemD journal (if available) +- **Custom paths**: Configurable additional log paths + +### Reliability Features +- **Local buffering**: Stores logs locally when server is unreachable +- **Automatic retry**: Reconnects automatically with exponential backoff +- **Compression**: Compresses logs before sending to save bandwidth +- **Secure transmission**: Optional TLS/SSL encryption +- **Backpressure handling**: Slows down when server is overwhelmed + +## Architecture + +### How It Works +1. Filebeat runs as a container with Docker socket access +2. Uses Docker API to stream logs from all containers +3. Monitors Docker API for container lifecycle events +4. Automatically starts collecting logs from new containers +5. Reads host system logs from mounted volumes +6. Ships all logs to configured Logstash/Elasticsearch endpoint +7. Maintains connection state and buffering information + +### Log Flow +``` +Docker Containers → Docker API → + ↘ + Filebeat → Logstash → Elasticsearch → Kibana + ↗ +System Logs (mounted volumes) → +``` + +### Why Docker API Instead of Log Files? +- **Logging driver flexibility**: Containers can use `local`, `json-file`, `journald`, or any driver +- **No log file management**: Don't need to worry about log rotation or file paths +- **Better performance**: Direct streaming without file I/O overhead +- **Consistent access**: Same method regardless of storage backend +- **Real-time streaming**: Get logs immediately as they're generated +- **Simplified permissions**: Only need Docker socket access + +## Minimum Configuration + +The template requires minimal configuration - server address and authentication: + +```bash +# Required - Server connection +LOGSERVER_HOST=192.168.1.100 +LOGSERVER_PORT=5044 + +# Required - Authentication (choose one method) +AUTH_MODE=mtls # Options: mtls, apikey, basic + +# For mTLS authentication +CLIENT_CERT_PATH=/certs/client.crt +CLIENT_KEY_PATH=/certs/client.key +CA_CERT_PATH=/certs/ca.crt + +# For API key authentication +API_KEY=your-api-key-here + +# For basic auth (not recommended) +USERNAME=filebeat +PASSWORD=changeme +``` + +## Configuration Options + +### Environment Variables (service.env) + +```bash +# REQUIRED: Log server connection +LOGSERVER_HOST=logserver.example.com +LOGSERVER_PORT=5044 + +# REQUIRED: Authentication method +AUTH_MODE=mtls # mtls, apikey, or basic + +# mTLS Authentication (if AUTH_MODE=mtls) +CLIENT_CERT_PATH=/certs/${HOSTNAME}.crt +CLIENT_KEY_PATH=/certs/${HOSTNAME}.key +CA_CERT_PATH=/certs/ca.crt +SSL_VERIFICATION_MODE=full + +# API Key Authentication (if AUTH_MODE=apikey) +API_KEY="" # Will be provided by logserver admin + +# Basic Authentication (if AUTH_MODE=basic) +USERNAME=filebeat +PASSWORD=changeme + +# Optional: Performance tuning +BULK_MAX_SIZE=2048 # Maximum batch size +WORKER_THREADS=1 # Number of worker threads +QUEUE_SIZE=4096 # Internal queue size +MAX_BACKOFF=60s # Maximum retry backoff + +# Optional: Filtering +EXCLUDE_CONTAINERS="" # Comma-separated container names to exclude +INCLUDE_CONTAINERS="" # Only include these containers (if set) +EXCLUDE_LABELS="" # Exclude containers with these labels +INCLUDE_LABELS="" # Only include containers with these labels + +# Optional: Additional log paths +CUSTOM_LOG_PATHS="" # Comma-separated additional paths to monitor + +# Optional: Resource limits +MAX_CPU=50 # Maximum CPU usage percentage +MAX_MEMORY=200MB # Maximum memory usage +``` + +## Collected Log Types + +### Docker Container Logs (via Docker API) +- **stdout/stderr**: All container output regardless of logging driver +- **Container metadata**: Name, ID, image, labels +- **Docker events**: Start, stop, die, kill events +- **Health check results**: If configured +- **Works with all logging drivers**: local, json-file, journald, syslog, etc. + +### System Logs +- **System messages**: Service starts/stops, errors +- **Authentication**: SSH logins, sudo usage +- **Kernel messages**: Hardware events, driver messages +- **Package management**: apt/yum operations +- **Cron jobs**: Scheduled task execution + +## Log Enrichment + +Logs are automatically enriched with: +- **Hostname**: Source host identification +- **Timestamp**: Precise event time with timezone +- **Log level**: Parsed from log content when possible +- **Container info**: For Docker logs +- **Process info**: PID, command for system logs +- **File path**: Source log file + +## Resource Requirements + +### Minimum +- CPU: 0.5 cores +- RAM: 128MB +- Storage: 1GB (for buffer) + +### Typical Usage +- CPU: 1-5% of one core +- RAM: 150-200MB +- Network: Varies with log volume +- Storage: Depends on buffer size + +## Installation + +### Prerequisites +1. A running logserver (ELK stack) +2. Network connectivity to logserver +3. Docker installed on host +4. Authentication credentials from logserver admin + +### Setup Authentication + +#### For mTLS (Recommended): +```bash +# Get client certificate from logserver admin +# They will run: dropshell exec logserver /scripts/generate-client-cert.sh $(hostname) +# Copy the generated certificate files to this client +mkdir -p /etc/dropshell/certs +# Copy ca.crt, client.crt, and client.key to /etc/dropshell/certs/ +``` + +#### For API Key: +```bash +# Get API key from logserver admin +# They will run: dropshell exec logserver /scripts/generate-api-key.sh $(hostname) +# Add the API key to service.env +``` + +### Deploy +```bash +# Configure authentication in service.env +dropshell install logclient +``` + +## Monitoring + +### Check Status +```bash +dropshell status logclient +``` + +### View Filebeat Logs +```bash +dropshell logs logclient +``` + +### Verify Connectivity +```bash +# Check if logs are being shipped +docker exec logclient-filebeat filebeat test output +``` + +### Monitor Metrics +```bash +# View Filebeat statistics +docker exec logclient-filebeat curl -s http://localhost:5066/stats +``` + +## Troubleshooting + +### No Logs Appearing on Server + +1. **Check connectivity** + ```bash + telnet $LOGSERVER_HOST $LOGSERVER_PORT + ``` + +2. **Verify Filebeat is running** + ```bash + dropshell status logclient + ``` + +3. **Check Filebeat logs** + ```bash + dropshell logs logclient | tail -50 + ``` + +4. **Test configuration** + ```bash + docker exec logclient-filebeat filebeat test config + ``` + +### High CPU Usage + +1. Reduce worker threads in service.env +2. Increase bulk_max_size to send larger batches +3. Add exclude filters for noisy containers + +### Missing Container Logs + +1. Verify Docker socket is mounted +2. Check container isn't in exclude list +3. Ensure Filebeat has permissions to Docker socket +4. Verify container is actually producing output +5. Check if container uses a supported logging driver + +### Buffer Full Errors + +1. Increase queue_size in service.env +2. Check network connectivity to server +3. Verify server isn't overwhelmed + +## Security Considerations + +1. **Authentication**: + - Always use mTLS or API keys in production + - Never use basic auth except for testing + - Store credentials securely + - Rotate certificates/keys regularly + +2. **Docker Socket Access**: + - Requires Docker socket access to read logs via API + - Understand security implications of socket access + - Consider read-only socket access if possible + +3. **Network Security**: + - All connections are TLS encrypted + - Verify server certificates + - Configure firewall rules appropriately + - Use private networks when possible + +4. **Data Protection**: + - Logs may contain sensitive data + - Filter sensitive information before shipping + - Exclude containers with sensitive data if needed + +5. **Resource Limits**: + - Set CPU and memory limits + - Monitor resource usage + - Prevent resource exhaustion attacks + +## Performance Tuning + +### For High-Volume Environments +```bash +# Increase workers and batch size +WORKER_THREADS=4 +BULK_MAX_SIZE=4096 +QUEUE_SIZE=8192 +``` + +### For Low-Resource Hosts +```bash +# Reduce resource usage +WORKER_THREADS=1 +BULK_MAX_SIZE=512 +MAX_MEMORY=100MB +MAX_CPU=25 +``` + +### Network Optimization +```bash +# Enable compression (CPU vs bandwidth tradeoff) +COMPRESSION_LEVEL=3 # 0-9, higher = more compression +``` + +## Integration with LogServer + +This template is designed to work seamlessly with the `logserver` template: + +1. Deploy logserver first +2. Note the logserver's IP/hostname +3. Configure logclient with server address +4. Logs automatically start flowing + +## Maintenance + +### Regular Tasks +- Monitor buffer usage +- Check for connection errors +- Review excluded/included containers +- Update Filebeat version + +### Logs Rotation +Filebeat handles log rotation automatically: +- Detects renamed/rotated files +- Continues reading from correct position +- Cleans up old file handles + +## Advanced Configuration + +### Custom Filebeat Configuration +Create a custom `filebeat.yml` in the config directory for advanced scenarios: +- Custom processors +- Additional inputs +- Complex filtering rules +- Multiple outputs + +### Docker Labels for Control +Control log collection per container: +```yaml +# In docker-compose.yml +services: + myapp: + logging: + driver: local # Can use any driver - Filebeat reads via API + options: + max-size: "10m" + max-file: "3" + labels: + - "filebeat.enable=false" # Exclude this container + - "filebeat.multiline.pattern=^\\[" # Custom multiline pattern +``` + +### Logging Driver Compatibility +The Docker API input works with all Docker logging drivers: +- **local**: Recommended for production (efficient, no file access needed) +- **json-file**: Traditional default driver +- **journald**: SystemD journal integration +- **syslog**: Forward to syslog +- **none**: Disables logging (Filebeat won't collect) + +You can use the `local` driver for better performance since Filebeat doesn't need to read files. \ No newline at end of file diff --git a/logclient/TODO.md b/logclient/TODO.md new file mode 100644 index 0000000..19f8209 --- /dev/null +++ b/logclient/TODO.md @@ -0,0 +1,294 @@ +# LogClient Template - Implementation TODO + +## Phase 1: Core Infrastructure (Priority 1) + +### Configuration Files +- [ ] Create `config/.template_info.env` with template metadata +- [ ] Create `config/service.env` with minimal required settings +- [ ] Define LOGSERVER_HOST and LOGSERVER_PORT variables +- [ ] Add AUTH_MODE variable (mtls, apikey, basic) +- [ ] Add certificate/key path variables for mTLS +- [ ] Add API_KEY variable for API key auth +- [ ] Add USERNAME/PASSWORD for basic auth +- [ ] Add optional performance and filtering variables +- [ ] Set sensible defaults where possible + +### Filebeat Configuration +- [ ] Create base `filebeat.yml` configuration template +- [ ] Configure Docker input using Docker API (not autodiscover with hints) +- [ ] Set containers.ids: ["*"] to collect from all containers +- [ ] Set up system log inputs for host logs +- [ ] Configure output to Logstash +- [ ] Add error handling and retry logic +- [ ] Set up local disk buffering +- [ ] Configure stream: "all" to get both stdout and stderr + +### Required Scripts +- [ ] Implement `install.sh` - Pull Filebeat image, configure auth, start +- [ ] Implement `uninstall.sh` - Stop and remove container (preserve config and certs) +- [ ] Implement `start.sh` - Start Filebeat with auth config and proper mounts +- [ ] Implement `stop.sh` - Gracefully stop Filebeat +- [ ] Implement `status.sh` - Check Filebeat health and auth status +- [ ] Create `setup-auth.sh` - Helper script to configure authentication + +## Phase 2: Docker API Log Collection (Priority 1) + +### Docker API Input Configuration +- [ ] Configure Docker input type (NOT autodiscover, use direct Docker input) +- [ ] Mount Docker socket (/var/run/docker.sock) with proper permissions +- [ ] Configure Docker API endpoint (unix:///var/run/docker.sock) +- [ ] Set up real-time log streaming from Docker daemon +- [ ] Enable collection from ALL logging drivers (local, json-file, journald, etc.) +- [ ] Configure since_time to get recent logs on startup + +### Container Metadata Extraction +- [ ] Extract container name, ID, image name, and image tag +- [ ] Map container labels to fields +- [ ] Handle docker-compose project names and service names +- [ ] Add container state information +- [ ] Include container environment variables (filtered) +- [ ] Handle container lifecycle events (start, stop, restart) + +### Container Filtering +- [ ] Implement include/exclude by container name patterns +- [ ] Add label-based filtering (containers.labels) +- [ ] Create ignore patterns for system containers +- [ ] Add support for custom filter expressions +- [ ] Configure combine_partial to handle partial log lines +- [ ] Document filtering examples with Docker API syntax + +## Phase 3: System Log Collection (Priority 1) + +### Log File Inputs +- [ ] Configure /var/log/syslog or /var/log/messages +- [ ] Add /var/log/auth.log or /var/log/secure +- [ ] Include /var/log/kern.log +- [ ] Monitor /var/log/dpkg.log or /var/log/yum.log +- [ ] Add custom log path support via environment variable + +### Journald Integration +- [ ] Detect if systemd/journald is available +- [ ] Configure journald input if present +- [ ] Set up unit filtering +- [ ] Extract systemd metadata +- [ ] Handle binary journal format + +### Log Parsing +- [ ] Configure syslog parsing +- [ ] Extract severity levels +- [ ] Parse timestamps correctly +- [ ] Handle different syslog formats +- [ ] Add timezone handling + +## Phase 4: Output Configuration (Priority 1) + +### Logstash Output +- [ ] Configure primary Logstash endpoint +- [ ] Set up connection parameters (timeout, retry) +- [ ] Configure bulk operations settings +- [ ] Add compression support +- [ ] Implement backpressure handling + +### Connection Management +- [ ] Configure automatic reconnection +- [ ] Set exponential backoff for retries +- [ ] Add connection pooling +- [ ] Configure keepalive settings +- [ ] Handle DNS resolution failures + +### Authentication Configuration (Priority 1 - CRITICAL) +- [ ] Implement mTLS authentication support +- [ ] Configure client certificate and key loading +- [ ] Add CA certificate validation +- [ ] Implement API key authentication +- [ ] Add basic auth as fallback option +- [ ] Create authentication mode selection logic +- [ ] Handle authentication failures gracefully +- [ ] Add certificate expiry checking +- [ ] Implement secure credential storage +- [ ] Document authentication setup process + +## Phase 5: Reliability Features (Priority 2) + +### Local Buffering +- [ ] Configure disk queue for reliability +- [ ] Set queue size limits +- [ ] Configure memory queue settings +- [ ] Add overflow handling +- [ ] Set up automatic cleanup of old events + +### Error Handling +- [ ] Add retry logic for failed sends +- [ ] Configure dead letter queue +- [ ] Add circuit breaker pattern +- [ ] Log transmission errors appropriately +- [ ] Add metrics for monitoring failures + +### Performance Optimization +- [ ] Configure worker count +- [ ] Set batch size for sending +- [ ] Add compression level setting +- [ ] Configure CPU and memory limits +- [ ] Optimize for high-volume scenarios + +## Phase 6: Optional Scripts (Priority 2) + +### Operational Scripts +- [ ] Implement `logs.sh` - Show Filebeat logs +- [ ] Implement `destroy.sh` - Complete removal +- [ ] Implement `ssh.sh` - Shell into Filebeat container +- [ ] Create `test.sh` - Test connectivity to server +- [ ] Add `metrics.sh` - Show Filebeat statistics + +### Diagnostic Scripts +- [ ] Create connectivity test script +- [ ] Add configuration validation script +- [ ] Create debug mode enabler +- [ ] Add log sampling script +- [ ] Create performance benchmark script + +## Phase 7: Monitoring & Health (Priority 2) + +### Health Checks +- [ ] Configure Filebeat HTTP endpoint +- [ ] Add Docker health check +- [ ] Monitor queue status +- [ ] Check connection to Logstash +- [ ] Track dropped events + +### Metrics Collection +- [ ] Enable Filebeat monitoring +- [ ] Export metrics endpoint +- [ ] Track events sent/failed +- [ ] Monitor resource usage +- [ ] Add performance counters + +### Status Reporting +- [ ] Implement detailed status in status.sh +- [ ] Show connection state +- [ ] Display queue status +- [ ] Report recent errors +- [ ] Show throughput metrics + +## Phase 8: Advanced Features (Priority 3) + +### Processors +- [ ] Add field renaming processor +- [ ] Configure drop_event conditions +- [ ] Add rate limiting processor +- [ ] Include fingerprinting for deduplication +- [ ] Add custom field enrichment + +### Multiline Handling +- [ ] Configure patterns for common languages +- [ ] Java stack trace handling +- [ ] Python traceback handling +- [ ] Go panic handling +- [ ] Custom pattern support via environment + +### Field Management +- [ ] Configure field inclusion/exclusion +- [ ] Add custom fields via environment +- [ ] Set up field type conversions +- [ ] Add timestamp parsing +- [ ] Configure field aliasing + +## Phase 9: Testing (Priority 3) + +### Unit Testing +- [ ] Test configuration generation +- [ ] Verify volume mounts +- [ ] Test environment variable substitution +- [ ] Validate filtering logic +- [ ] Test error conditions + +### Integration Testing +- [ ] Test with logserver template +- [ ] Verify Docker log collection +- [ ] Test system log collection +- [ ] Validate SSL connectivity +- [ ] Test reconnection scenarios +- [ ] Verify buffering during outages + +### Load Testing +- [ ] Test with high log volume +- [ ] Measure resource usage +- [ ] Test queue overflow handling +- [ ] Verify rate limiting +- [ ] Benchmark throughput + +## Phase 10: Documentation (Priority 3) + +### User Documentation +- [ ] Create README.txt for dropshell +- [ ] Document all configuration options +- [ ] Add troubleshooting guide +- [ ] Create quick start guide +- [ ] Add FAQ section + +### Configuration Examples +- [ ] Minimal configuration example +- [ ] High-volume configuration +- [ ] Secure SSL configuration +- [ ] Filtered configuration +- [ ] Custom paths configuration + +### Integration Guides +- [ ] Integration with logserver +- [ ] Docker Compose examples +- [ ] Kubernetes DaemonSet example +- [ ] Swarm mode configuration +- [ ] Custom application integration + +## Phase 11: Production Readiness (Priority 4) + +### Security Hardening +- [ ] Run as non-root user where possible +- [ ] Minimize container capabilities +- [ ] Add secrets management +- [ ] Configure log sanitization +- [ ] Add audit logging + +### Updates & Maintenance +- [ ] Add update notification +- [ ] Create upgrade script +- [ ] Add configuration migration +- [ ] Document breaking changes +- [ ] Create rollback procedure + +### Compatibility +- [ ] Test with different Filebeat versions +- [ ] Verify Docker API compatibility +- [ ] Test on different Linux distributions +- [ ] Validate with various log formats +- [ ] Ensure Logstash version compatibility + +## Notes + +### Design Principles +1. **Minimal configuration**: Just needs LOGSERVER_HOST to work +2. **Docker API access**: Use Docker API for driver-independent log collection +3. **Automatic discovery**: Find all container logs without manual configuration +4. **Reliability first**: Never lose logs, buffer locally if needed +5. **Low overhead**: Minimal resource usage on host +6. **Non-intrusive**: No changes to existing containers needed +7. **Driver flexibility**: Allow containers to use any logging driver (especially `local`) + +### Key Requirements +- Must work with zero configuration beyond server address +- Must use Docker API input, not file-based collection +- Must support all Docker logging drivers (local, json-file, etc.) +- Must handle Docker socket permissions properly +- Must be resilient to network failures +- Must not impact host performance significantly +- Must preserve configuration on uninstall + +### Testing Checklist +- [ ] Validates with dropshell test-template +- [ ] Connects to logserver successfully +- [ ] Collects Docker logs automatically +- [ ] Collects system logs properly +- [ ] Handles server downtime gracefully +- [ ] Reconnects automatically +- [ ] Resource usage stays within limits +- [ ] Uninstall preserves configuration \ No newline at end of file diff --git a/logclient/config/.template_info.env b/logclient/config/.template_info.env new file mode 100644 index 0000000..64ec230 --- /dev/null +++ b/logclient/config/.template_info.env @@ -0,0 +1,17 @@ +# Template identifier - MUST match the directory name +TEMPLATE=logclient + +# Requirements +REQUIRES_HOST_ROOT=false # No root access on host needed +REQUIRES_DOCKER=true # Docker is required +REQUIRES_DOCKER_ROOT=false # Docker root privileges not specifically needed + +# Docker image settings +IMAGE_REGISTRY="docker.elastic.co" +IMAGE_REPO="beats/filebeat" +IMAGE_TAG="7.17.23" + +# Volume definitions +CONFIG_VOLUME="${CONTAINER_NAME}_config" +DATA_VOLUME="${CONTAINER_NAME}_data" +CERTS_VOLUME="${CONTAINER_NAME}_certs" \ No newline at end of file diff --git a/logclient/config/service.env b/logclient/config/service.env new file mode 100644 index 0000000..3128180 --- /dev/null +++ b/logclient/config/service.env @@ -0,0 +1,44 @@ +# Service identification +CONTAINER_NAME=logclient-filebeat + +# Docker image tag override (optional) +IMAGE_TAG="7.17.23" + +# REQUIRED: Log server connection +LOGSERVER_HOST= +LOGSERVER_PORT=5044 + +# REQUIRED: Authentication method +AUTH_MODE=mtls # mtls, apikey, or basic + +# mTLS Authentication (if AUTH_MODE=mtls) +CLIENT_CERT_PATH=/certs/client.crt +CLIENT_KEY_PATH=/certs/client.key +CA_CERT_PATH=/certs/ca.crt +SSL_VERIFICATION_MODE=full + +# API Key Authentication (if AUTH_MODE=apikey) +API_KEY="" # Will be provided by logserver admin + +# Basic Authentication (if AUTH_MODE=basic) +USERNAME=filebeat +PASSWORD=changeme + +# Performance tuning +BULK_MAX_SIZE=2048 # Maximum batch size +WORKER_THREADS=1 # Number of worker threads +QUEUE_SIZE=4096 # Internal queue size +MAX_BACKOFF=60s # Maximum retry backoff + +# Filtering +EXCLUDE_CONTAINERS="" # Comma-separated container names to exclude +INCLUDE_CONTAINERS="" # Only include these containers (if set) +EXCLUDE_LABELS="" # Exclude containers with these labels +INCLUDE_LABELS="" # Only include containers with these labels + +# Additional log paths +CUSTOM_LOG_PATHS="" # Comma-separated additional paths to monitor + +# Resource limits +MAX_CPU=50 # Maximum CPU usage percentage +MAX_MEMORY=200MB # Maximum memory usage \ No newline at end of file diff --git a/logclient/install.sh b/logclient/install.sh new file mode 100755 index 0000000..8e751b9 --- /dev/null +++ b/logclient/install.sh @@ -0,0 +1,62 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Check required environment variables +_check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" "LOGSERVER_HOST" "LOGSERVER_PORT" "AUTH_MODE" + +# Validate authentication configuration +case "$AUTH_MODE" in + mtls) + _check_required_env_vars "CLIENT_CERT_PATH" "CLIENT_KEY_PATH" "CA_CERT_PATH" + if [ ! -f "$CLIENT_CERT_PATH" ]; then + _die "Client certificate not found at $CLIENT_CERT_PATH" + fi + if [ ! -f "$CLIENT_KEY_PATH" ]; then + _die "Client key not found at $CLIENT_KEY_PATH" + fi + if [ ! -f "$CA_CERT_PATH" ]; then + _die "CA certificate not found at $CA_CERT_PATH" + fi + ;; + apikey) + _check_required_env_vars "API_KEY" + if [ -z "$API_KEY" ]; then + _die "API_KEY is empty. Please get an API key from the logserver administrator" + fi + ;; + basic) + _check_required_env_vars "USERNAME" "PASSWORD" + echo "WARNING: Basic authentication is not recommended for production" + ;; + *) + _die "Invalid AUTH_MODE: $AUTH_MODE. Must be one of: mtls, apikey, basic" + ;; +esac + +# Check Docker is available +_check_docker_installed || _die "Docker test failed" + +# Test connectivity to logserver +echo "Testing connectivity to logserver at ${LOGSERVER_HOST}:${LOGSERVER_PORT}..." +nc -zv "$LOGSERVER_HOST" "$LOGSERVER_PORT" 2>/dev/null || echo "WARNING: Cannot connect to logserver. Will retry when container starts." + +# Pull the Docker image +docker pull "$IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG" || _die "Failed to pull Filebeat image" + +# Stop any existing container +bash ./stop.sh || true + +# Remove old container +_remove_container "$CONTAINER_NAME" || true + +# Generate Filebeat configuration +echo "Generating Filebeat configuration..." +bash ./scripts/generate-config.sh || _die "Failed to generate configuration" + +# Start the new container +bash ./start.sh || _die "Failed to start Filebeat" + +echo "Installation of ${CONTAINER_NAME} complete" +echo "Collecting logs from Docker API and shipping to ${LOGSERVER_HOST}:${LOGSERVER_PORT}" +echo "Authentication mode: ${AUTH_MODE}" \ No newline at end of file diff --git a/logclient/start.sh b/logclient/start.sh new file mode 100755 index 0000000..cd447fb --- /dev/null +++ b/logclient/start.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" "IMAGE_REGISTRY" "IMAGE_REPO" "IMAGE_TAG" + +# Create Docker command +cmd="docker run -d \ + --name $CONTAINER_NAME \ + --restart unless-stopped \ + --user root \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/log:/var/log:ro \ + -v ${CONFIG_VOLUME}:/usr/share/filebeat/config:ro \ + -v ${DATA_VOLUME}:/usr/share/filebeat/data \ + -v ${CERTS_VOLUME}:/usr/share/filebeat/certs:ro \ + -e LOGSERVER_HOST=${LOGSERVER_HOST} \ + -e LOGSERVER_PORT=${LOGSERVER_PORT} \ + -e AUTH_MODE=${AUTH_MODE} \ + $IMAGE_REGISTRY/$IMAGE_REPO:$IMAGE_TAG \ + filebeat -e -strict.perms=false \ + -c /usr/share/filebeat/config/filebeat.yml" + +# Start the container +eval "$cmd" || _die "Failed to start Filebeat container" + +echo "Filebeat started successfully" \ No newline at end of file diff --git a/logclient/status.sh b/logclient/status.sh new file mode 100755 index 0000000..45bfe3e --- /dev/null +++ b/logclient/status.sh @@ -0,0 +1,31 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +# Check if container exists +if ! docker ps -a --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + echo "Unknown" + exit 0 +fi + +# Check container state +STATE=$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null) +case "$STATE" in + running) + # Additional check: verify connection to logserver + if docker logs "$CONTAINER_NAME" 2>&1 | tail -20 | grep -q "ERROR"; then + echo "Error" + else + echo "Running" + fi + ;; + exited|stopped) + echo "Stopped" + ;; + restarting|paused) + echo "Error" + ;; + *) + echo "Unknown" + ;; +esac \ No newline at end of file diff --git a/logclient/stop.sh b/logclient/stop.sh new file mode 100755 index 0000000..7e9b8c6 --- /dev/null +++ b/logclient/stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +docker stop "$CONTAINER_NAME" 2>/dev/null || true + +echo "Filebeat stopped" \ No newline at end of file diff --git a/logclient/uninstall.sh b/logclient/uninstall.sh new file mode 100755 index 0000000..1df1674 --- /dev/null +++ b/logclient/uninstall.sh @@ -0,0 +1,17 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +# Stop the container +bash ./stop.sh || _die "Failed to stop container" + +# Remove the container +_remove_container "$CONTAINER_NAME" || _die "Failed to remove container" + +# CRITICAL: Never remove data volumes in uninstall.sh! +# Data volumes must be preserved for potential reinstallation +# Configuration and certificates must be preserved + +echo "Uninstallation of ${CONTAINER_NAME} complete" +echo "Note: Configuration and certificates have been preserved." +echo "To remove all data, use destroy.sh" \ No newline at end of file diff --git a/logserver/README.md b/logserver/README.md new file mode 100644 index 0000000..2bd0ce2 --- /dev/null +++ b/logserver/README.md @@ -0,0 +1,279 @@ +# Dropshell LogServer Template + +A comprehensive centralized logging solution using the ELK Stack (Elasticsearch, Logstash, Kibana) for receiving, processing, and visualizing logs from multiple hosts. + +## Overview + +This template deploys a full-featured ELK stack that: +- Receives logs from multiple sources via Beats protocol +- Stores and indexes logs in Elasticsearch +- Provides powerful search and visualization through Kibana +- Supports automatic log parsing and enrichment +- Handles Docker container logs and system logs from clients + +## Architecture + +### Components + +1. **Elasticsearch** (7.17.x) + - Distributed search and analytics engine + - Stores and indexes all log data + - Provides fast full-text search capabilities + - Single-node configuration for simplicity (can be scaled) + +2. **Logstash** (7.17.x) + - Log processing pipeline + - Receives logs from Filebeat clients + - Parses and enriches log data + - Routes logs to appropriate Elasticsearch indices + +3. **Kibana** (7.17.x) + - Web UI for log exploration and visualization + - Create dashboards and alerts + - Real-time log streaming + - Advanced search queries + +## Features + +### Minimum Configuration Design +- Auto-discovery of log formats +- Pre-configured dashboards for common services +- Automatic index lifecycle management +- Built-in parsing for Docker and syslog formats +- Zero-configuration client connectivity + +### Log Processing +- Automatic timestamp extraction +- Docker metadata enrichment (container name, image, labels) +- Syslog parsing with severity levels +- JSON log support +- Multi-line log handling (stacktraces, etc.) +- Grok pattern matching for common formats + +### Security & Performance +- **Mutual TLS (mTLS)** authentication for client connections +- **API key authentication** as an alternative to certificates +- **Per-client authentication** with unique keys/certificates +- **SSL/TLS encryption** for all client connections +- **Basic authentication** for Kibana web access +- **IP whitelisting** for additional security +- Index lifecycle management for storage optimization +- Automatic old log cleanup +- Resource limits to prevent overconsumption + +## Port Configuration + +- **5601**: Kibana Web UI (HTTP/HTTPS with authentication) +- **9200**: Elasticsearch REST API (HTTP) - internal only +- **5044**: Logstash Beats input (TCP/TLS) - authenticated client connections +- **514**: Syslog input (UDP/TCP) - optional, unauthenticated +- **24224**: Fluentd forward input - optional Docker logging driver + +## Storage Requirements + +- **Minimum**: 10GB for basic operation +- **Recommended**: 50GB+ depending on log volume +- **Log Retention**: Default 30 days (configurable) + +## Client Authentication + +### Authentication Methods + +1. **Mutual TLS (mTLS) - Recommended** + - Each client gets a unique certificate signed by the server's CA + - Strongest security with mutual authentication + - Automatic certificate validation + +2. **API Keys** + - Each client gets a unique API key + - Simpler to manage than certificates + - Good for environments where certificate management is difficult + +3. **Basic Auth (Not Recommended)** + - Shared username/password + - Least secure, only for testing + +### Client Configuration + +Clients using the `logclient` template will: +1. Authenticate using provided credentials (cert/key or API key) +2. Establish encrypted TLS connection +3. Ship all Docker container logs +4. Ship system logs (syslog, auth, kernel) +5. Maintain connection with automatic reconnection +6. Buffer logs locally during network outages + +## Dashboard Features + +### Pre-configured Dashboards +- **System Overview**: Overall health and log volume metrics +- **Docker Containers**: Container-specific logs and metrics +- **Error Analysis**: Aggregated error logs from all sources +- **Security Events**: Authentication and access logs +- **Application Logs**: Parsed application-specific logs + +### Search Capabilities +- Full-text search across all logs +- Filter by time range, host, container, severity +- Save and share search queries +- Export search results + +## Resource Requirements + +### Minimum +- CPU: 2 cores +- RAM: 4GB +- Storage: 10GB + +### Recommended +- CPU: 4+ cores +- RAM: 8GB+ +- Storage: 50GB+ SSD + +## Configuration Options + +### Environment Variables (service.env) + +```bash +# Elasticsearch settings +ES_HEAP_SIZE=2g +ES_MAX_MAP_COUNT=262144 + +# Logstash settings +LS_HEAP_SIZE=1g +LS_PIPELINE_WORKERS=2 + +# Kibana settings +KIBANA_PASSWORD=changeme +KIBANA_BASE_PATH=/ + +# Log retention +LOG_RETENTION_DAYS=30 +LOG_MAX_SIZE_GB=50 + +# Authentication Mode +AUTH_MODE=mtls # Options: mtls, apikey, basic +ENABLE_TLS=true + +# mTLS Settings (if AUTH_MODE=mtls) +CA_CERT_PATH=/certs/ca.crt +SERVER_CERT_PATH=/certs/server.crt +SERVER_KEY_PATH=/certs/server.key +CLIENT_CERT_REQUIRED=true + +# API Key Settings (if AUTH_MODE=apikey) +API_KEYS_PATH=/config/api-keys.yml + +# Network Security +ALLOWED_IPS="" # Comma-separated list, empty = all +``` + +## Usage + +### Installation +```bash +dropshell install logserver +``` + +### Generate Client Credentials + +#### For mTLS Authentication: +```bash +# Generate client certificate for a new host +dropshell exec logserver /scripts/generate-client-cert.sh hostname +# This creates hostname.crt and hostname.key files +``` + +#### For API Key Authentication: +```bash +# Generate API key for a new client +dropshell exec logserver /scripts/generate-api-key.sh hostname +# Returns an API key to configure in the client +``` + +### Access Kibana +Navigate to `https://:5601` in your browser. + +Default credentials: +- Username: `elastic` +- Password: `changeme` (change in service.env) + +### View Logs +```bash +dropshell logs logserver +``` + +### Backup +```bash +dropshell backup logserver +``` + +## Troubleshooting + +### Common Issues + +1. **Elasticsearch failing to start** + - Check vm.max_map_count: `sysctl vm.max_map_count` (should be 262144+) + - Verify sufficient memory available + +2. **No logs appearing in Kibana** + - Check Logstash is receiving data: port 5044 should be open + - Verify client connectivity + - Check index patterns in Kibana + +3. **High memory usage** + - Adjust heap sizes in service.env + - Configure index lifecycle management + - Reduce retention period + +## Integration + +This template is designed to work seamlessly with the `logclient` template. Simply: +1. Deploy this logserver +2. Deploy logclient on each host you want to monitor +3. Configure logclient with the logserver address +4. Logs will automatically start flowing + +## Security Considerations + +1. **Authentication Setup** + - Use mTLS for production environments + - Generate unique credentials for each client + - Rotate certificates/keys regularly + - Store credentials securely + +2. **Network Security** + - Always use TLS encryption for client connections + - Configure IP whitelisting when possible + - Use firewall rules to restrict access + - Consider VPN or private networks + +3. **Access Control** + - Change default Kibana password immediately + - Create read-only users for viewing logs + - Implement role-based access control (RBAC) + - Audit access logs regularly + +4. **Data Protection** + - Regular backups of Elasticsearch indices + - Encrypt data at rest (optional) + - Monitor disk usage to prevent data loss + - Implement log retention policies + +## Maintenance + +### Daily Tasks +- Monitor disk usage +- Check for failed log shipments +- Review error dashboards + +### Weekly Tasks +- Verify all clients are reporting +- Check index health +- Review and optimize slow queries + +### Monthly Tasks +- Update ELK stack components +- Archive old indices +- Review retention policies +- Performance tuning based on usage patterns \ No newline at end of file diff --git a/logserver/TODO.md b/logserver/TODO.md new file mode 100644 index 0000000..2b73a35 --- /dev/null +++ b/logserver/TODO.md @@ -0,0 +1,244 @@ +# LogServer Template - Implementation TODO + +## Phase 1: Core Infrastructure (Priority 1) + +### Configuration Files +- [ ] Create `config/.template_info.env` with template metadata +- [ ] Create `config/service.env` with user-configurable settings +- [ ] Define all required environment variables (ports, passwords, heap sizes) +- [ ] Set appropriate default values for zero-config experience + +### Docker Compose Setup +- [ ] Create `docker-compose.yml` with ELK stack services +- [ ] Configure Elasticsearch single-node setup +- [ ] Configure Logstash with Beats input pipeline +- [ ] Configure Kibana with Elasticsearch connection +- [ ] Set up proper networking between services +- [ ] Define named volumes for data persistence +- [ ] Configure health checks for each service + +### Required Scripts +- [ ] Implement `install.sh` - Pull images, create volumes, start services +- [ ] Implement `uninstall.sh` - Stop and remove containers (preserve volumes!) +- [ ] Implement `start.sh` - Start all ELK services with docker-compose +- [ ] Implement `stop.sh` - Gracefully stop all services +- [ ] Implement `status.sh` - Check health of all three services + +## Phase 2: Logstash Configuration (Priority 1) + +### Input Configuration +- [ ] Configure Beats input on port 5044 with TLS/SSL +- [ ] Set up mutual TLS (mTLS) authentication +- [ ] Configure client certificate validation +- [ ] Add API key authentication option +- [ ] Implement IP whitelisting +- [ ] Add Syslog input on port 514 (UDP/TCP) - unauthenticated +- [ ] Add Docker Fluentd input on port 24224 (optional) + +### Filter Pipeline +- [ ] Create Docker log parser (extract container metadata) +- [ ] Create Syslog parser (RFC3164 and RFC5424) +- [ ] Add JSON parser for structured logs +- [ ] Implement multiline pattern for stack traces +- [ ] Add timestamp extraction and normalization +- [ ] Create field enrichment (add host metadata) +- [ ] Implement conditional routing based on log type + +### Output Configuration +- [ ] Configure Elasticsearch output with index patterns +- [ ] Set up index templates for different log types +- [ ] Configure index lifecycle management (ILM) + +## Phase 3: Elasticsearch Setup (Priority 1) + +### System Configuration +- [ ] Set appropriate heap size defaults (ES_HEAP_SIZE) +- [ ] Configure vm.max_map_count requirement check +- [ ] Set up single-node discovery settings +- [ ] Configure data persistence volume +- [ ] Set up index templates for: + - [ ] Docker logs (docker-*) + - [ ] System logs (syslog-*) + - [ ] Application logs (app-*) + - [ ] Error logs (errors-*) + +### Index Management +- [ ] Configure ILM policies for log rotation +- [ ] Set retention period (default 30 days) +- [ ] Configure max index size limits +- [ ] Set up automatic cleanup of old indices +- [ ] Create snapshot repository configuration + +## Phase 4: Kibana Configuration (Priority 2) + +### Initial Setup +- [ ] Configure Kibana with Elasticsearch URL +- [ ] Set up basic authentication +- [ ] Configure server base path +- [ ] Set appropriate memory limits + +### Pre-built Dashboards +- [ ] Create System Overview dashboard +- [ ] Create Docker Containers dashboard +- [ ] Create Error Analysis dashboard +- [ ] Create Security Events dashboard +- [ ] Create Host Metrics dashboard + +### Saved Searches +- [ ] Error logs across all sources +- [ ] Authentication events +- [ ] Container lifecycle events +- [ ] Slow queries/performance issues +- [ ] Critical system events + +### Index Patterns +- [ ] Configure docker-* pattern +- [ ] Configure syslog-* pattern +- [ ] Configure app-* pattern +- [ ] Configure filebeat-* pattern + +## Phase 5: Optional Scripts (Priority 2) + +### Operational Scripts +- [ ] Implement `logs.sh` - Show logs from all ELK services +- [ ] Implement `backup.sh` - Snapshot Elasticsearch indices +- [ ] Implement `restore.sh` - Restore from snapshots +- [ ] Implement `destroy.sh` - Complete removal including volumes +- [ ] Implement `ports.sh` - Display all exposed ports +- [ ] Implement `ssh.sh` - Shell into specific container + +### Helper Scripts +- [ ] Create `_volumes.sh` for volume management helpers +- [ ] Add health check script for all services +- [ ] Create performance tuning script +- [ ] Add certificate generation script for SSL + +## Phase 6: Security Features (Priority 1 - CRITICAL) + +### Certificate Authority Setup +- [ ] Create CA certificate and key for signing client certs +- [ ] Generate server certificate for Logstash +- [ ] Create certificate generation script for clients +- [ ] Set up certificate storage structure +- [ ] Implement certificate rotation mechanism + +### mTLS Authentication +- [ ] Configure Logstash for mutual TLS +- [ ] Set up client certificate validation +- [ ] Create client certificate generation script +- [ ] Implement certificate revocation list (CRL) +- [ ] Add certificate expiry monitoring + +### API Key Authentication +- [ ] Create API key generation script +- [ ] Configure Logstash to accept API keys +- [ ] Implement API key storage (encrypted) +- [ ] Add API key rotation mechanism +- [ ] Create API key revocation process + +### Network Security +- [ ] Implement IP whitelisting in Logstash +- [ ] Configure firewall rules +- [ ] Set up rate limiting +- [ ] Add connection throttling +- [ ] Implement DDoS protection + +### Kibana Security +- [ ] Configure Kibana HTTPS +- [ ] Set up basic authentication +- [ ] Create user management scripts +- [ ] Implement session management +- [ ] Add audit logging + +## Phase 7: Performance & Optimization (Priority 3) + +### Resource Management +- [ ] Configure CPU limits for each service +- [ ] Set memory limits appropriately +- [ ] Add swap handling configuration +- [ ] Configure JVM options files +- [ ] Add performance monitoring + +### Optimization +- [ ] Configure pipeline workers +- [ ] Set batch sizes for optimal throughput +- [ ] Configure queue sizes +- [ ] Add caching configuration +- [ ] Optimize index refresh intervals + +## Phase 8: Testing & Documentation (Priority 3) + +### Testing +- [ ] Test installation process +- [ ] Test uninstall (verify volume preservation) +- [ ] Test log ingestion from sample client +- [ ] Test all dashboard functionality +- [ ] Test backup and restore procedures +- [ ] Load test with high log volume +- [ ] Test failover and recovery + +### Documentation +- [ ] Create README.txt for dropshell format +- [ ] Document all configuration options +- [ ] Add troubleshooting guide +- [ ] Create quick start guide +- [ ] Document upgrade procedures +- [ ] Add performance tuning guide + +## Phase 9: Integration Testing (Priority 3) + +### With LogClient +- [ ] Test automatic discovery +- [ ] Verify log flow from client to server +- [ ] Test reconnection scenarios +- [ ] Verify all log types are parsed correctly +- [ ] Test SSL communication +- [ ] Measure end-to-end latency + +### Compatibility Testing +- [ ] Test with different Docker versions +- [ ] Test on various Linux distributions +- [ ] Verify with different log formats +- [ ] Test with high-volume producers +- [ ] Validate resource usage + +## Phase 10: Production Readiness (Priority 4) + +### Monitoring & Alerting +- [ ] Add Elasticsearch monitoring +- [ ] Configure disk space alerts +- [ ] Set up index health monitoring +- [ ] Add performance metrics collection +- [ ] Create alert rules in Kibana + +### Maintenance Features +- [ ] Add automatic update check +- [ ] Create maintenance mode +- [ ] Add data export functionality +- [ ] Create migration scripts +- [ ] Add configuration validation + +## Notes + +### Design Principles +1. **Minimum configuration**: Should work with just `dropshell install logserver` +2. **Data safety**: Never delete volumes in uninstall.sh +3. **Non-interactive**: All scripts must run without user input +4. **Idempotent**: Scripts can be run multiple times safely +5. **Clear feedback**: Provide clear status and error messages + +### Dependencies +- Docker and Docker Compose +- Sufficient system resources (4GB+ RAM recommended) +- Network connectivity for clients +- Persistent storage for logs + +### Testing Checklist +- [ ] All required scripts present and executable +- [ ] Template validates with dropshell test-template +- [ ] Services start and connect properly +- [ ] Logs flow from client to Kibana +- [ ] Data persists across container restarts +- [ ] Uninstall preserves data volumes +- [ ] Resource limits are enforced +- [ ] Error handling works correctly \ No newline at end of file diff --git a/logserver/config/.template_info.env b/logserver/config/.template_info.env new file mode 100644 index 0000000..2c60810 --- /dev/null +++ b/logserver/config/.template_info.env @@ -0,0 +1,17 @@ +# Template identifier - MUST match the directory name +TEMPLATE=logserver + +# Requirements +REQUIRES_HOST_ROOT=false # No root access on host needed +REQUIRES_DOCKER=true # Docker is required +REQUIRES_DOCKER_ROOT=false # Docker root privileges not specifically needed + +# Docker compose used for ELK stack +USES_DOCKER_COMPOSE=true + +# Volume definitions for persistence +DATA_VOLUME="${CONTAINER_NAME}_elasticsearch_data" +LOGSTASH_VOLUME="${CONTAINER_NAME}_logstash_data" +KIBANA_VOLUME="${CONTAINER_NAME}_kibana_data" +CERTS_VOLUME="${CONTAINER_NAME}_certs" +CONFIG_VOLUME="${CONTAINER_NAME}_config" \ No newline at end of file diff --git a/logserver/config/service.env b/logserver/config/service.env new file mode 100644 index 0000000..75ed386 --- /dev/null +++ b/logserver/config/service.env @@ -0,0 +1,46 @@ +# Service identification +CONTAINER_NAME=logserver + +# Elasticsearch settings +ES_VERSION=7.17.23 +ES_HEAP_SIZE=2g +ES_MAX_MAP_COUNT=262144 + +# Logstash settings +LS_VERSION=7.17.23 +LS_HEAP_SIZE=1g +LS_PIPELINE_WORKERS=2 + +# Kibana settings +KIBANA_VERSION=7.17.23 +KIBANA_PASSWORD=changeme +KIBANA_BASE_PATH=/ + +# Ports +KIBANA_PORT=5601 +LOGSTASH_BEATS_PORT=5044 +LOGSTASH_SYSLOG_PORT=514 + +# Log retention +LOG_RETENTION_DAYS=30 +LOG_MAX_SIZE_GB=50 + +# Authentication Mode +AUTH_MODE=mtls # Options: mtls, apikey, basic +ENABLE_TLS=true + +# mTLS Settings (if AUTH_MODE=mtls) +CA_CERT_PATH=/certs/ca.crt +SERVER_CERT_PATH=/certs/server.crt +SERVER_KEY_PATH=/certs/server.key +CLIENT_CERT_REQUIRED=true + +# API Key Settings (if AUTH_MODE=apikey) +API_KEYS_PATH=/config/api-keys.yml + +# Network Security +ALLOWED_IPS="" # Comma-separated list, empty = all + +# Resource limits +MAX_CPU_PERCENT=80 +MAX_MEMORY=4GB \ No newline at end of file diff --git a/logserver/install.sh b/logserver/install.sh new file mode 100755 index 0000000..056e641 --- /dev/null +++ b/logserver/install.sh @@ -0,0 +1,62 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Check required environment variables +_check_required_env_vars "CONTAINER_NAME" "ES_VERSION" "LS_VERSION" "KIBANA_VERSION" + +# Check Docker and Docker Compose are available +_check_docker_installed || _die "Docker test failed" +which docker-compose >/dev/null 2>&1 || _die "docker-compose is not installed" + +# Check vm.max_map_count for Elasticsearch +current_max_map_count=$(sysctl -n vm.max_map_count 2>/dev/null || echo 0) +if [ "$current_max_map_count" -lt 262144 ]; then + echo "WARNING: vm.max_map_count is too low ($current_max_map_count)" + echo "Elasticsearch requires at least 262144" + echo "Please run: sudo sysctl -w vm.max_map_count=262144" + echo "And add to /etc/sysctl.conf to persist" + _die "System configuration needs adjustment" +fi + +# Stop any existing containers +bash ./stop.sh || true + +# Remove old containers +docker-compose down --remove-orphans 2>/dev/null || true + +# Pull the Docker images +echo "Pulling ELK stack images..." +docker pull docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION} || _die "Failed to pull Elasticsearch" +docker pull docker.elastic.co/logstash/logstash:${LS_VERSION} || _die "Failed to pull Logstash" +docker pull docker.elastic.co/kibana/kibana:${KIBANA_VERSION} || _die "Failed to pull Kibana" + +# Generate certificates if using mTLS +if [ "$AUTH_MODE" = "mtls" ]; then + bash ./scripts/generate-ca.sh || _die "Failed to generate CA certificate" + bash ./scripts/generate-server-cert.sh || _die "Failed to generate server certificate" +fi + +# Start the ELK stack +echo "Starting ELK stack..." +docker-compose up -d --build || _die "Failed to start ELK stack" + +# Wait for services to be ready +echo "Waiting for services to start..." +sleep 10 + +# Check status +bash ./status.sh || _die "Services failed to start properly" + +echo "Installation of ${CONTAINER_NAME} complete" +echo "" +echo "Kibana UI: http://$(hostname -I | awk '{print $1}'):${KIBANA_PORT}" +echo "Username: elastic" +echo "Password: ${KIBANA_PASSWORD}" +echo "" +echo "Logstash listening on port ${LOGSTASH_BEATS_PORT} for Filebeat clients" +if [ "$AUTH_MODE" = "mtls" ]; then + echo "Authentication: mTLS (generate client certs with ./scripts/generate-client-cert.sh)" +elif [ "$AUTH_MODE" = "apikey" ]; then + echo "Authentication: API Keys (generate with ./scripts/generate-api-key.sh)" +fi \ No newline at end of file diff --git a/logserver/start.sh b/logserver/start.sh new file mode 100755 index 0000000..c91fa3b --- /dev/null +++ b/logserver/start.sh @@ -0,0 +1,17 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +echo "Starting ELK stack..." +docker-compose up -d || _die "Failed to start ELK stack" + +# Wait for services to be ready +echo "Waiting for services to start..." +sleep 5 + +# Check if services are running +if docker-compose ps | grep -q "Up"; then + echo "ELK stack started successfully" +else + _die "Failed to start ELK stack services" +fi \ No newline at end of file diff --git a/logserver/status.sh b/logserver/status.sh new file mode 100755 index 0000000..5a21c0a --- /dev/null +++ b/logserver/status.sh @@ -0,0 +1,22 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +# Check if docker-compose services exist and are running +if ! docker-compose ps 2>/dev/null | grep -q "${CONTAINER_NAME}"; then + echo "Unknown" + exit 0 +fi + +# Check individual service status +elasticsearch_status=$(docker-compose ps elasticsearch 2>/dev/null | grep -c "Up") +logstash_status=$(docker-compose ps logstash 2>/dev/null | grep -c "Up") +kibana_status=$(docker-compose ps kibana 2>/dev/null | grep -c "Up") + +if [ "$elasticsearch_status" -eq 1 ] && [ "$logstash_status" -eq 1 ] && [ "$kibana_status" -eq 1 ]; then + echo "Running" +elif [ "$elasticsearch_status" -eq 0 ] && [ "$logstash_status" -eq 0 ] && [ "$kibana_status" -eq 0 ]; then + echo "Stopped" +else + echo "Error" +fi \ No newline at end of file diff --git a/logserver/stop.sh b/logserver/stop.sh new file mode 100755 index 0000000..626161b --- /dev/null +++ b/logserver/stop.sh @@ -0,0 +1,8 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +echo "Stopping ELK stack..." +docker-compose stop || true + +echo "ELK stack stopped" \ No newline at end of file diff --git a/logserver/uninstall.sh b/logserver/uninstall.sh new file mode 100755 index 0000000..893614e --- /dev/null +++ b/logserver/uninstall.sh @@ -0,0 +1,16 @@ +#!/bin/bash +source "${AGENT_PATH}/common.sh" +_check_required_env_vars "CONTAINER_NAME" + +# Stop the containers +bash ./stop.sh || _die "Failed to stop containers" + +# Remove the containers +docker-compose down --remove-orphans || _die "Failed to remove containers" + +# CRITICAL: Never remove data volumes in uninstall.sh! +# Data volumes must be preserved for potential reinstallation +# Only destroy.sh should remove volumes, and it must be explicit + +echo "Uninstallation of ${CONTAINER_NAME} complete" +echo "Note: Data volumes have been preserved. To remove all data, use destroy.sh" \ No newline at end of file