[Feature] Elasticagent + conditionally Elasticsearch+Kibana in deployment

docker-compose-local.yml

@@ -54,6 +54,7 @@ x-domains_env:
   PRODUCT_DOMAIN: &productdomain "fake.${SERVER_DOMAIN:-localmaeher.pvarki.fi}"
   TAK_DOMAIN: &takdomain "tak.${SERVER_DOMAIN:-localmaeher.pvarki.fi}"
   TAK_RMAPI_PORT: &takapiport ${TAK_RMAPI_PORT:-4626}
+  KIBANA_DOMAIN: &kibanadomain "kibana.${SERVER_DOMAIN:-localmaeher.pvarki.fi}" # local Kibana
   DNS_RESOLVER_IP: &dnsresolver ${DNS_RESOLVER_IP:-127.0.0.11}  # Must be able to resolve docker internal names
   OCSCP_RESPONDER:  &publicocsp "https://${SERVER_DOMAIN:-localmaeher.pvarki.fi}:${NGINX_HTTPS_PORT:-4439}/ca/ocsp"  # The public URL
 
@@ -473,6 +474,111 @@ services:
       - kraftwerk_shared_fake:/pvarki
       - kwinit_data:/data/persistent
 
+############################
+# BEGIN Elastic & Kibana   #
+############################
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+    container_name: elasticsearch
+    environment:
+      - node.name=elasticsearch
+      - discovery.type=single-node
+      - network.host=0.0.0.0
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es_data:/usr/share/elasticsearch/data
+    networks:
+      - loggingnet
+    ports:
+      - "9200:9200"
+    restart: unless-stopped
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.15.3
+    container_name: kibana
+    environment:
+      - SERVER_NAME=kibana
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - XPACK_SECURITY_ENABLED=false
+    depends_on:
+      - elasticsearch
+    networks:
+      - loggingnet
+    expose:
+      - "5601"
+    restart: unless-stopped
+
+  kibananginx:
+    <<: *nginxbuildinfo
+    volumes:
+      - nginx_templates:/nginx_templates
+      - ca_public:/ca_public
+      - le_certs:/le_certs
+    environment:
+      NGINX_HOST: "kibana.${SERVER_DOMAIN}"
+      NGINX_HTTP_PORT: "80"
+      NGINX_HTTPS_PORT: "5601"
+      NGINX_UPSTREAM: "kibana"
+      NGINX_UPSTREAM_PORT: "5601"
+      NGINX_CERT_NAME: "rasenmaeher"
+      CFSSL_OCSP_BIND_PORT: *oscpport
+      NGINX_OCSP_UPSTREAM: *ocsphost
+      DNS_RESOLVER_IP: *dnsresolver
+      NGINX_TEMPLATE_DIR: "templates_kibana"
+    networks:
+      - loggingnet
+      - ocspnet
+    ports:
+      - "5601:5601"
+    depends_on:
+      kibana:
+        condition: service_started
+      nginx_templates:
+        condition: service_completed_successfully
+      ocsp:
+        condition: service_healthy
+      cfssl:
+        condition: service_healthy
+    healthcheck:
+      test: 'curl -s localhost:5666/healthcheck || exit 1'
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    restart: unless-stopped
+
+  elastic-agent:
+    image: docker.elastic.co/beats/elastic-agent:8.15.3
+    container_name: elastic-agent
+    user: root
+    environment:
+      ELASTIC_AGENT_ENABLED: "false"  # Disable Fleet enrollment in local mode
+      ELASTIC_AGENT_ID: "local-agent"  # Assign a unique ID
+      ELASTIC_AGENT_ENROLLMENT_TOKEN: "dummy-token"
+      ELASTIC_AGENT_FLEET_URL: "http://dummy-fleet-url"
+      ELASTICSEARCH_HOSTS: "http://elasticsearch:9200"
+      LOG_INPUT_ENABLED: "true"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
+      - ./elastic-agent-config/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml
+    networks:
+      - loggingnet
+    restart: unless-stopped
+
+
+##########################
+# END Elastic & Kibana   #
+##########################
+
 ######################
 # Begin: Fakeproduct #
 ######################
@@ -780,6 +886,7 @@ networks:
   dbnet:
   intranet:
   taknet:
+  loggingnet:
 
 volumes:
   kraftwerk_data:
@@ -798,3 +905,4 @@ volumes:
   takrmapi_data:
   rmui_files:
   nginx_templates:
+  es_data:

docker-compose.yml

@@ -52,6 +52,7 @@ x-domains_env:
   PRODUCT_DOMAIN: &productdomain "fake.${SERVER_DOMAIN:?domain must be defined}"
   TAK_DOMAIN: &takdomain "tak.${SERVER_DOMAIN:?domain must be defined}"
   TAK_RMAPI_PORT: &takapiport ${TAK_RMAPI_PORT:-4626}
+  KIBANA_DOMAIN: &kibanadomain ${KIBANA_DOMAIN:-} # not defined, if enrolled to Elastic fleet
   DNS_RESOLVER_IP: &dnsresolver ${DNS_RESOLVER_IP:-127.0.0.11}  # Must be able to resolve docker internal names
   OCSCP_RESPONDER:  &publicocsp "https://${SERVER_DOMAIN:?domain must be defined}:${NGINX_HTTPS_PORT:-443}/ca/ocsp"  # The public URL
 
@@ -445,6 +446,115 @@ services:
       start_period: 5s
     restart: unless-stopped
 
+  elastic-agent:
+    image: docker.elastic.co/beats/elastic-agent:8.15.3
+    container_name: elastic-agent
+    user: root
+    environment:
+      ELASTIC_AGENT_ENABLED: ${ELASTIC_AGENT_ENABLED-false} # Unless set to true, the agent will not start
+      ELASTIC_AGENT_ENROLLMENT_TOKEN: "${ELASTIC_AGENT_ENROLLMENT_TOKEN:-dummy-token}"
+      ELASTIC_AGENT_FLEET_URL: "${ELASTIC_AGENT_FLEET_URL:-http://dummy-fleet-url}"
+      ELASTICSEARCH_HOSTS: "http://elasticsearch:9200"
+      LOG_INPUT_ENABLED: "true"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /:/hostfs:ro
+      - ./elastic-agent-config/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml
+    networks:
+      - loggingnet
+    restart: unless-stopped
+
+  ##########################
+  # BEGIN profile: kibana  #
+  ##########################
+
+    elasticsearch:
+      image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+      container_name: elasticsearch
+      environment:
+        - node.name=elasticsearch
+        - discovery.type=single-node
+        - network.host=0.0.0.0
+        - bootstrap.memory_lock=true
+        - xpack.security.enabled=false
+        - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      ulimits:
+        memlock:
+          soft: -1
+          hard: -1
+      volumes:
+        - es_data:/usr/share/elasticsearch/data
+      networks:
+        - loggingnet
+      ports:
+        - "9200:9200"
+      restart: unless-stopped
+      profiles:
+        - kibana
+
+    kibana:
+      image: docker.elastic.co/kibana/kibana:8.15.3
+      container_name: kibana
+      environment:
+        - SERVER_NAME=kibana
+        - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+        - XPACK_SECURITY_ENABLED=false
+      depends_on:
+        - elasticsearch
+      networks:
+        - loggingnet
+      expose:
+        - "5601"
+      restart: unless-stopped
+      profiles:
+        - kibana
+
+  kibananginx:
+    <<: *nginxbuildinfo
+    volumes:
+      - nginx_templates:/nginx_templates
+      - ca_public:/ca_public
+      - le_certs:/le_certs
+    environment:
+      NGINX_HOST: "kibana.${SERVER_DOMAIN}"
+      NGINX_HTTP_PORT: "80"
+      NGINX_HTTPS_PORT: "5601"
+      NGINX_UPSTREAM: "kibana"
+      NGINX_UPSTREAM_PORT: "5601"
+      NGINX_CERT_NAME: "kibana"
+      CFSSL_OCSP_BIND_PORT: *oscpport
+      NGINX_OCSP_UPSTREAM: *ocsphost
+      DNS_RESOLVER_IP: *dnsresolver
+      NGINX_TEMPLATE_DIR: "templates_kibana"
+    networks:
+      - loggingnet
+      - ocspnet
+    ports:
+      - "5601:5601"
+    depends_on:
+      kibana:
+        condition: service_started
+      nginx_templates:
+        condition: service_completed_successfully
+      ocsp:
+        condition: service_healthy
+      cfssl:
+        condition: service_healthy
+    healthcheck:
+      test: 'curl -s localhost:5666/healthcheck || exit 1'
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    restart: unless-stopped
+    profiles:
+      - kibana
+
+  ##########################
+  # END profile: kibana  #
+  ##########################
+
   kwinit:  # Mostly to make sure it's built
     image: pvarki/kw_product_init:1.0.0-d${RELEASE_TAG:-1.5.1}${DOCKER_TAG_EXTRA:-}
     build:
@@ -701,6 +811,7 @@ networks:
   dbnet:
   intranet:
   taknet:
+  loggingnet:
 
 volumes:
   kraftwerk_data:
@@ -719,3 +830,4 @@ volumes:
   takrmapi_data:
   rmui_files:
   nginx_templates:
+  es_data:

elastic-agent-config/elastic-agent.yml

@@ -0,0 +1,30 @@
+agent:
+  id: ${ELASTIC_AGENT_ID}
+
+# Fleet enrollment configuration
+fleet:
+  enabled: ${ELASTIC_AGENT_ENABLED}
+  enrollment_token: ${ELASTIC_AGENT_ENROLLMENT_TOKEN}
+  hosts:
+    - ${ELASTIC_AGENT_FLEET_URL}
+
+# Standalone configuration
+outputs:
+  default:
+    type: elasticsearch
+    hosts:
+      - ${ELASTICSEARCH_HOSTS:-http://elasticsearch:9200}
+
+inputs:
+  - type: docker
+    id: docker-logs
+    enabled: ${LOG_INPUT_ENABLED:-true}
+    streams:
+      - containers.ids:
+          - '*'  # Collect logs from all containers
+    processors:
+      - add_docker_metadata: ~
+    # Enable the Docker module
+    modules:
+      - name: docker
+        enabled: true

example_env.sh

@@ -11,4 +11,9 @@ export MW_LE_EMAIL="example@example.com"
 export MW_LE_TEST="true"  # switch to false when you are ready for production
 export TAKSERVER_CERT_PASS="KissaKoira123!AlpakkaMursu"  # used for the JKS
 export TAK_CA_PASS="AlpakkaMursu!KissaKoira123"  # used for the JKS
+export KIBANA_DOMAIN="kibana.${SERVER_DOMAIN}" # Set kibana domain, if you want to use kibana locally
 export VITE_ASSET_SET="${VITE_ASSET_SET:-neutral}"  # used RMUI to define asset sets (logos, etc).
+export ELASTIC_AGENT_ENABLED="false" # If you want to enable ElasticAgent for fleet configuration, set true
+export ELASTIC_AGENT_ENROLLMENT_TOKEN= # Token for Elastic fleet enrolment
+export ELASTIC_AGENT_FLEET_URL= #Url for Elasti fleet enrolment
+export LOG_INPUT_ENABLED="true"

nginx/templates_kibana/default.conf.template

@@ -0,0 +1,34 @@
+ssl_certificate /le_certs/${NGINX_CERT_NAME}/fullchain.pem;
+ssl_certificate_key /le_certs/${NGINX_CERT_NAME}/privkey.pem;
+
+include /etc/nginx/includes/le_common_settings.conf;
+
+server {
+    server_name ${NGINX_HOST};
+
+    # HTTPS configuration
+    listen ${NGINX_HTTPS_PORT} ssl;
+
+    ssl_client_certificate /ca_public/ca_chain.pem;
+    ssl_verify_client      on;
+    ssl_ocsp leaf;
+    ssl_ocsp_responder http://${NGINX_OCSP_UPSTREAM}:${CFSSL_OCSP_BIND_PORT};
+    resolver ${DNS_RESOLVER_IP} ipv6=off;
+    #ssl_crl /ca_public/crl.pem;
+    ssl_verify_depth 3;
+
+    location / {
+        if ($ssl_client_verify != SUCCESS) {
+            return 401;
+        }
+        proxy_pass  http://${NGINX_UPSTREAM}:${NGINX_UPSTREAM_PORT};
+        proxy_redirect                      off;
+        proxy_set_header  Host              $http_host;
+        proxy_set_header  X-Real-IP         $remote_addr;
+        proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header  X-Forwarded-Proto $scheme;
+        proxy_read_timeout                  900;
+        proxy_set_header  X-ClientCert-DN   $ssl_client_s_dn;
+        proxy_set_header  X-ClientCert-Serial   $ssl_client_serial;
+    }
+}