From 150646d1e973475e3c5d12342d2d183211e43e6f Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Tue, 15 Mar 2022 17:27:56 +1100 Subject: [PATCH] feat: make analytics and logging stack optional (#99) --- .env | 23 ++++--- compose-files/metricbeat.yml | 20 ------ docker-compose.yml | 127 +++++++++++++++++++++++++++++++++-- placeos | 101 ++++++++++++++++++---------- scripts/metricbeat | 8 --- scripts/start-services | 17 ----- 6 files changed, 199 insertions(+), 97 deletions(-) delete mode 100644 compose-files/metricbeat.yml delete mode 100755 scripts/metricbeat delete mode 100755 scripts/start-services diff --git a/.env b/.env index e08cc4f..cd43be5 100644 --- a/.env +++ b/.env @@ -38,12 +38,23 @@ PLACE_METRICS_ROUTE=monitor ELASTIC_HOST=elastic ELASTIC_PORT=9200 -ELASTIC_VERSION=7.16.2 +ELASTIC_VERSION=7.10.2 ETCD_HOST=etcd ETCD_PORT=2379 ETCD_VERSION=3.5.1 +REDIS_URL=redis://redis:6379 + +RETHINKDB_DB=place_development +RETHINKDB_HOST=rethink +RETHINKDB_PORT=28015 +RETHINKDB_VERSION=2.4 + +# Analytics variables + +ENABLE_ANALYTICS=false + # INFLUX_USER=placeos # INFLUX_PASSWORD=development @@ -52,20 +63,14 @@ INFLUX_HOST=http://influxdb:8086 INFLUX_ORG=PlaceOS INFLUX_RETENTION=4w -REDIS_URL=redis://redis:6379 - -RETHINKDB_DB=place_development -RETHINKDB_HOST=rethink -RETHINKDB_PORT=28015 -RETHINKDB_VERSION=2.4 - # Staff API variables POSTGRES_USER=placeos POSTGRES_PASSWORD=development -# Monitor Node variables +# Logging variables +ENABLE_KIBANA=false LOGSTASH_HOST=logstash LOGSTASH_PORT=12201 KIBANA_PORT=443 diff --git a/compose-files/metricbeat.yml b/compose-files/metricbeat.yml deleted file mode 100644 index e7ac66a..0000000 --- a/compose-files/metricbeat.yml +++ /dev/null @@ -1,20 +0,0 @@ - # Gets metrics from host machine and send to elastic - metricbeat: - image: docker.elastic.co/beats/metricbeat-oss:${ELASTIC_VERSION:-7.6} - hostname: $MONITOR_HOSTNAME - container_name: metricbeat - user: root - networks: - placeos: - volumes: - - /proc:/hostfs/proc:ro - - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro - - /:/hostfs:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml - cap_add: - - SYS_PTRACE - - DAC_READ_SEARCH - command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] - restart: unless-stopped - << : *logging-env diff --git a/docker-compose.yml b/docker-compose.yml index 706950a..774aac1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ x-jwt-public-key-env: &jwt-public-key-env .env.public_key x-secret-key-env: &secret-key-env .env.secret_key x-elastic-client-env: &elastic-client-env + ELASTIC_HOST: ${ELASTIC_HOST:-elastic} + ELASTIC_PORT: ${ELASTIC_PORT:-9200} ES_HOST: ${ELASTIC_HOST:-elastic} ES_PORT: ${ELASTIC_PORT:-9200} @@ -191,9 +193,12 @@ services: PLACE_LOADER_WWW: www source: - image: docker.io/placeos/source:${PLACE_SOURCE_TAG:-nightly} + image: placeos/source:${PLACE_SOURCE_TAG:-nightly} + profiles: + - analytics restart: always container_name: source + hostname: source <<: *std-network <<: *std-logging depends_on: @@ -235,6 +240,7 @@ services: postgres: # Database used by Staff API image: postgres:${POSTGRES_VERSION:-13-alpine} container_name: postgres + hostname: postgres restart: unless-stopped <<: *std-network <<: *std-logging @@ -248,6 +254,7 @@ services: staff: # Staff API image: placeos/staff-api:${PLACE_STAFF_API_TAG:-nightly} container_name: staff + hostname: staff restart: unless-stopped <<: *std-network <<: *std-logging @@ -323,12 +330,13 @@ services: # Resources elastic: - image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.9.1} + image: blacktop/elasticsearch:${ELASTIC_VERSION:-7.10.2} restart: always container_name: elastic hostname: elastic healthcheck: test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health + start_period: 1m <<: *std-network <<: *std-logging volumes: @@ -356,11 +364,13 @@ services: influxdb: image: influxdb:${INFLUXDB_IMAGE_TAG:-2.0.8-alpine} - container_name: influx + profiles: + - analytics restart: always + container_name: influx + hostname: influx <<: *std-network <<: *std-logging - hostname: influx healthcheck: test: influx bucket list volumes: @@ -368,11 +378,14 @@ services: source: influx-data target: /root/.influxdbv2 command: "--reporting-disabled" - + chronograf: image: chronograf:${CHRONOGRAF_IMAGE_TAG:-1.9} - container_name: chronograf + profiles: + - analytics restart: always + container_name: chronograf + hostname: chronograf <<: *std-network <<: *std-logging env_file: @@ -394,6 +407,8 @@ services: mosquitto: image: iegomez/mosquitto-go-auth:${MOSQUITTO_IMAGE_TAG:-latest} + profiles: + - analytics restart: always container_name: mosquitto hostname: mosquitto @@ -466,3 +481,103 @@ services: target: /data/rethinkdb_data environment: TZ: $TZ + + # Aggregates logs and forwards them to Elasticsearch. + logstash: + image: blacktop/logstash:${ELASTIC_VERSION:-7.10.2} + profiles: + - kibana + restart: always + container_name: logstash + hostname: logstash + << : *std-network + << : *std-logging + depends_on: + - validate-logstash-config + volumes: + - ${PWD}/config/logstash/config:/config + - ${PWD}/config/logstash/patterns:/opt/logstash/extra_patterns + command: logstash -f /config + + # Run 'docker-compose run --rm validate-logstash-config' to quickly check the logstash config. + validate-logstash-config: + image: blacktop/logstash:${ELASTIC_VERSION:-7.10.2} + profiles: + - kibana + restart: "no" + container_name: validate-logstash + << : *std-network + << : *std-logging + volumes: + - ${PWD}/config/logstash/config:/config + command: logstash -t -f /config + + # Sends all container json-file logs to logstash + logspout: + image: vincit/logspout-gelf:3.2.6-alpine + profiles: + - kibana + restart: unless-stopped + container_name: logspout + hostname: logspout + << : *std-network + << : *std-logging + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: gelf://${LOGSTASH_HOST}:${LOGSTASH_PORT} + + kibana: + image: blacktop/kibana:${ELASTIC_VERSION:-7.10.2} + profiles: + - kibana + restart: always + container_name: kibana + hostname: kibana + << : *std-network + << : *std-logging + environment: + <<: *elastic-client-env + NODE_OPTIONS: "--max-old-space-size=200" # fixes memory leak (https://github.com/elastic/kibana/issues/5170) + HTTPS_METHOD: "nohttp" + ELASTICSEARCH_HOSTS: "http://${ELASTIC_HOST}:${ELASTIC_PORT}" + SERVER_BASEPATH: "/${PLACE_METRICS_ROUTE}" + SERVER_REWRITEBASEPATH: "true" + SERVER_PUBLICBASEURL: "https://${PLACE_DOMAIN}/${PLACE_METRICS_ROUTE}" + + # Takes care of piling up Elasticsearch indices/logs. Can do many other things as well. + # Set up a cron job that runs "docker-compose run --rm curator --config /config.yml /action-file.yml" every once in a while. + curator: + image: bobrik/curator:5.8.1 + profiles: + - kibana + container_name: curator + hostname: curator + << : *std-network + << : *std-logging + volumes: + - ${PWD}/config/curator/action-file.yml:/action-file.yml + - ${PWD}/config/curator/config.yml:/config.yml + + # Gets metrics from host machine and send to elastic + metricbeat: + image: elastic/metricbeat:${ELASTIC_VERSION:-7.10.2} + profiles: + - metricbeat + restart: unless-stopped + container_name: metricbeat + hostname: metricbeat + user: root + << : *std-network + << : *std-logging + environment: + <<: *elastic-client-env + volumes: + - /proc:/hostfs/proc:ro + - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro + - /:/hostfs:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ${PWD}/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml + cap_add: + - SYS_PTRACE + - DAC_READ_SEARCH + command: ["metricbeat", "-e", "--strict.perms=false", "-system.hostfs=/hostfs", "-E", "output.elasticsearch.hosts=[$ELASTIC_HOST:$ELASTIC_PORT]"] diff --git a/placeos b/placeos index 09c666d..d4f859f 100755 --- a/placeos +++ b/placeos @@ -75,6 +75,17 @@ logfile="${base_path}/.logs/$(date +"%Y%m%d%H%M").log" COMPOSE_PROJECT_NAME=placeos +# Helpers +################################################################################################### + +unknown_argument() ( + unknown_arg="${1}" + usage="${2}" + [ -n "${unknown_arg}" ] && echo -e "░░░ ${red}Unknown option:${reset} ${unknown_arg}" + eval "${usage}" + exit 1 +) + abort() ( echo -e "░░░ ${red}${1}${reset}" echo "░░░ Logs can be found in ${logfile}" @@ -102,6 +113,9 @@ run_or_abort() { fi } +# Start +################################################################################################### + hard_reset() ( # TODO: drop influxdb tables # TODO: clear redis @@ -134,14 +148,18 @@ Arguments: --password PASSWORD Password for created admin account. [default: development] --domain DOMAIN Domain to configure. [default: localhost:8443] --application APP Application to configure. [default: backoffice] + --analytics Set-up analytics stack, including MQTT & InfluxDB & Chronograf + --kibana Set-up Kibana and Elastic stack. -v, --verbose Write logs to STDOUT in addition to the log file. -h, --help Display this message. EOF ) start_environment() ( - SERVICES=('') + PROFILES="" hard_reset=false + enable_analytics=false + enable_kibana=false email_argument="" password_argument="" domain_argument="" @@ -177,6 +195,12 @@ start_environment() ( application_arguement="${1}" shift ;; + --analytics) + enable_analytics=true + ;; + --kibana) + enable_kibana=true + ;; -v | --verbose) VERBOSE="true" ;; @@ -185,12 +209,7 @@ start_environment() ( exit 0 ;; *) - if [ -n "${command}" ]; then - echo -e "░░░ ${red}Unknown option:${reset} ${command}" - else - start_environment__usage - exit 1 - fi + unknown_argument "${command}" "start_environment__usage" ;; esac done @@ -199,8 +218,6 @@ start_environment() ( load_environment - # Override .env arguments with CLI arguments - echo "░░░ Starting PlaceOS <${PLACEOS_TAG}>" [ $VERBOSE == "false" ] && echo "░░░ For detailed logging, run \`tail -f ${logfile}\`" @@ -257,35 +274,48 @@ start_environment() ( echo "" done - # run_or_abort \ - # "${base_path}/scripts/metricbeat" \ - # "Checking Host OS..." \ - # "Error occurred while checking Host OS." - # Write the email so as to not prompt the user again. echo "PLACE_EMAIL=${PLACE_EMAIL}" >"${EMAIL_ENV}" # TODO: use init check instead of writing the password. echo "PLACE_PASSWORD=${PLACE_PASSWORD}" >>"${EMAIL_ENV}" + if [[ $ENABLE_ANALYTICS == "true" ]] || [[ $enable_analytics == "true" ]]; then + PROFILES+=" --profile analytics" + fi + if [[ -d "${base_path}/.htpasswd-kibana" ]]; then echo "░░░ Detected malformed auth file. Cleaning up" rm -r "${base_path}/.htpasswd-kibana" fi + if [[ $ENABLE_KIBANA == "true" ]] || [[ $enable_kibana == "true" ]]; then + PROFILES+=" --profile kibana" + if [[ $(uname) == "Linux" ]]; then + PROFILES+=" --profile metricbeat" + fi + fi + run_or_abort \ "${base_path}/scripts/generate-secrets" \ "Generating secrets..." \ "Failed to generate secrets." run_or_abort \ - "${base_path}/scripts/start-services ${SERVICES[@]}" \ + "docker-compose ${PROFILES} pull -q" \ + "Pulling images..." \ + "Failed to pull images." + + run_or_abort \ + "docker-compose ${PROFILES} up -d" \ "Bringing up services..." \ "Failed to start services." - run_or_abort \ - "${base_path}/scripts/init-influxdb" \ - "Configuring InfluxDB..." \ - "Failed to configure InfluxDB." + if [[ $PROFILES == *"analytics"* ]]; then + run_or_abort \ + "${base_path}/scripts/init-influxdb" \ + "Configuring InfluxDB..." \ + "Failed to configure InfluxDB." + fi [ ${hard_reset} == "true" ] && hard_reset @@ -298,6 +328,9 @@ start_environment() ( echo "░░░ $PLACE_EMAIL:$PLACE_PASSWORD" ) +# Stop +################################################################################################### + stop_environment__usage() ( cat < [help|...] [arguments...] @@ -383,6 +414,9 @@ task() ( ./scripts/run-sam-task ${PARAMS} ) +# Update +################################################################################################### + update_environment__usage() ( cat <>docker-compose.yml - fi -fi diff --git a/scripts/start-services b/scripts/start-services deleted file mode 100755 index 3b57e68..0000000 --- a/scripts/start-services +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -COMPOSE_FILES=('-f ./docker-compose.yml') - -for arg in "$@"; do - case $arg in - -e | --elk) - COMPOSE_FILES+=('-f ./compose-files/elk/docker-compose.yml') - ;; - esac -done - -docker-compose ${COMPOSE_FILES[@]} pull --quiet - -docker-compose ${COMPOSE_FILES[@]} up --detach --remove-orphans