diff --git a/.gitignore b/.gitignore index 9f09176..caee421 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ target coverage VERSION output.log +es-passwords.txt diff --git a/Rakefile b/Rakefile index 2ad0e2e..41ec051 100644 --- a/Rakefile +++ b/Rakefile @@ -9,6 +9,11 @@ task :all_sources do sh 'docker ps -a --format "table {{.ID}}\t{{.Status}}\t{{.Names}}\t{{.Ports}}"' end +desc 'Generate the certs Docker volume' +task :certs do + sh 'docker-compose -f docker-compose-create-certs.yml run --rm create_certs' +end + desc 'Clean some generated files' task :clean do %w[ @@ -23,22 +28,32 @@ task :clean do ].each { |f| FileUtils.rm_rf(Dir.glob(f)) } end -desc 'Stop the entire EFK stack, any additional sources and the minikube cluster' + +desc 'Start the client instances Kibana and ElasticHQ (Elasticsearch cluster must be up and running first)' +task :clients do + trap('SIGINT') do + puts 'Cancelled Kibana launch...' + exit + end + sh './scripts/start-clients.sh' +end + +desc 'Kill the entire EFK stack, any additional sources and the minikube cluster' task :down do - sh './scripts/stop-efk.sh || true' + sh './scripts/down-es-cluster.sh || true' end -desc 'Start the EFK stack components (including elasticHQ)' -task :efk do +desc 'Start the Elasticsearch cluster (including elasticHQ)' +task :elasticsearch do trap('SIGINT') do - puts 'Cancelled EFK stack launch...' + puts 'Cancelled Elasticsearch cluster launch...' exit end - sh './scripts/start-efk.sh' + sh './scripts/start-es-cluster.sh' end desc 'Run ALL the rake tasks: clean test and build' -task everything: %w[down clean style test efk k8s all_sources] +task everything: %w[down clean style test elasticsearch kibana k8s all_sources] desc 'Start the Kubernetes Minikube components' task :k8s do @@ -59,6 +74,15 @@ task :logs do sh 'docker-compose logs -f' end +task :passwords do + sh ' + docker exec elasticsearch-master /bin/bash \ + -c "bin/elasticsearch-setup-passwords auto \ + --batch \ + --url https://elasticsearch:9200" > es-passwords.txt + ' +end + desc 'Start the Prometheus stack component' task :prometheus do trap('SIGINT') do @@ -93,6 +117,11 @@ task :start, :source do |_task, args| sh "./scripts/start-source.sh #{args[:source]}" end +desc 'Stop the EFK cluster' +task :stop do + sh './scripts/stop-es-cluster.sh' +end + desc 'Run all style checks' task style: %w[rubocop] diff --git a/docker-compose-clients.yml b/docker-compose-clients.yml new file mode 100644 index 0000000..c4fccd3 --- /dev/null +++ b/docker-compose-clients.yml @@ -0,0 +1,47 @@ +version: '3.6' + +services: + kibana: + build: + context: ./kibana + args: + VERSION_EFK: ${VERSION_EFK} + container_name: kibana + environment: + ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_KIBANA_PASSWORD} + SERVER_SSL_KEYPASSPHRASE: ${KEY_PASSPHRASE} + hostname: kibana + networks: + - cluster + ports: + - 5601:5601 + volumes: + - certs:/usr/share/kibana/config/certificates + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml + + elastichq: + build: + context: ./elasticHQ + args: + VERSION_ELASTICHQ: ${VERSION_ELASTICHQ} + container_name: elastichq + environment: + HQ_DEFAULT_URL: https://elastic:${ELASTICSEARCH_ELASTIC_PASSWORD}@elasticsearch:9200 + HQ_ENABLE_SSL: "True" + HQ_CA_CERTS: /usr/share/elastichq/ca/ca.crt + HQ_VERIFY_CERTS: "False" + hostname: elastichq + networks: + - cluster + ports: + - 5000:5000 + volumes: + - certs:/usr/share/elastichq + +networks: + cluster: + driver: bridge + +volumes: + certs: + driver: local diff --git a/docker-compose-create-certs.yml b/docker-compose-create-certs.yml index 318ae3a..82046bd 100644 --- a/docker-compose-create-certs.yml +++ b/docker-compose-create-certs.yml @@ -30,12 +30,12 @@ services: - certs:/certs - ./elasticsearch/certs/instances.yml:/usr/share/elasticsearch/instances.yml networks: - - efk + - cluster volumes: certs: driver: local networks: - efk: + cluster: driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index f0fbf31..80cfabf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,10 @@ services: environment: - ${ES_JAVA_OPTS:-/dev/null} - KEY_PASSPHRASE=${KEY_PASSPHRASE} - healthcheck: - test: curl --cacert $CERTS_DIR/ca.crt -s https://elasticsearch:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi - interval: 30s - timeout: 10s - retries: 5 + - CERTS_DIR=${CERTS_DIR} hostname: elasticsearch networks: - - efk + - cluster ports: - "9200:9200" - "9300:9300" @@ -33,8 +29,7 @@ services: - certs:$CERTS_DIR - elasticsearch_master:/usr/share/elasticsearch/data - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh - - ./elasticsearch/config/master/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml + - ./elasticsearch/config/master.yml:/usr/share/elasticsearch/config/elasticsearch.yml elasticsearch-hot: build: @@ -48,7 +43,7 @@ services: - KEY_PASSPHRASE=${KEY_PASSPHRASE} hostname: elasticsearch-hot networks: - - efk + - cluster ulimits: nofile: soft: 65536 @@ -60,8 +55,7 @@ services: - certs:$CERTS_DIR - elasticsearch_hot:/usr/share/elasticsearch/data - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh - - ./elasticsearch/config/hot/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml + - ./elasticsearch/config/hot.yml:/usr/share/elasticsearch/config/elasticsearch.yml elasticsearch-warm: build: @@ -75,7 +69,7 @@ services: - KEY_PASSPHRASE=${KEY_PASSPHRASE} hostname: elasticsearch-warm networks: - - efk + - cluster ulimits: nofile: soft: 65536 @@ -87,8 +81,7 @@ services: - certs:$CERTS_DIR - elasticsearch_warm:/usr/share/elasticsearch/data - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh - - ./elasticsearch/config/warm/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml + - ./elasticsearch/config/warm.yml:/usr/share/elasticsearch/config/elasticsearch.yml # fluentd: # build: @@ -103,7 +96,7 @@ services: # - ./fluentd/fluentd.properties # hostname: fluentd # networks: -# - efk +# - cluster # ports: # - "5140:5140" # - "5140:5140/udp" @@ -116,50 +109,9 @@ services: # volumes: # - ./fluentd/config:/fluentd/etc # - ./fluentd/certs:/fluentd/certs -# - kibana: - build: - context: ./kibana - args: - VERSION_EFK: ${VERSION_EFK} - container_name: kibana - depends_on: - - elasticsearch - environment: - ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD} - SERVER_SSL_KEYPASSPHRASE: ${KEY_PASSPHRASE} - hostname: kibana - networks: - - efk - ports: - - 5601:5601 - volumes: - - certs:/usr/share/kibana/config/certificates - - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml - -# elastichq: -# build: -# context: ./elasticHQ -# args: -# VERSION_ELASTICHQ: ${VERSION_ELASTICHQ} -# container_name: elastichq -# depends_on: -# - elasticsearch -# environment: -# HQ_DEFAULT_URL: https://elasticsearch:9200 -# HQ_ENABLE_SSL: "True" -# HQ_CA_CERTS: /usr/share/elastichq/ca/ca.crt -# HQ_VERIFY_CERTS: "False" -# hostname: elastichq -# networks: -# - efk -# ports: -# - 5000:5000 -# volumes: -# - certs:/usr/share/elastichq networks: - efk: + cluster: driver: bridge volumes: diff --git a/elastichq/Dockerfile b/elastichq/Dockerfile index 4ac0e3d..2ec09a2 100644 --- a/elastichq/Dockerfile +++ b/elastichq/Dockerfile @@ -6,6 +6,4 @@ LABEL Description="ElasticHQ instance" LABEL Version="2.0.0" EXPOSE 5000 - -HEALTHCHECK --interval=5s --timeout=2s --retries=15 \ - CMD nc -z localhost 5000 || exit 1 +HEALTHCHECK --interval=5s --timeout=2s --retries=15 CMD nc -z localhost 5000 || exit 1 diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 1d7b3fb..f9d9b98 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -6,6 +6,4 @@ LABEL Description="Elasticsearch instance" LABEL Version="2.0.0" EXPOSE 9200 9300 - -HEALTHCHECK --interval=5s --timeout=2s --retries=15 \ - CMD curl --silent --fail localhost:9200/_cluster/health || exit 1 +HEALTHCHECK --interval=5s --timeout=2s --retries=15 CMD nc -z elasticsearch 9200 || exit 1 diff --git a/elasticsearch/config/hot/elasticsearch.yml b/elasticsearch/config/hot.yml similarity index 100% rename from elasticsearch/config/hot/elasticsearch.yml rename to elasticsearch/config/hot.yml diff --git a/elasticsearch/config/master/elasticsearch.yml b/elasticsearch/config/master.yml similarity index 100% rename from elasticsearch/config/master/elasticsearch.yml rename to elasticsearch/config/master.yml diff --git a/elasticsearch/config/readonlyrest.yml b/elasticsearch/config/readonlyrest.yml deleted file mode 100644 index 57ae42b..0000000 --- a/elasticsearch/config/readonlyrest.yml +++ /dev/null @@ -1,94 +0,0 @@ -readonlyrest: - ssl: - enable: false - # put the keystore in the same dir with elasticsearch.yml -# keystore_file: "keystore.jks" -# keystore_pass: readonlyrest -# key_pass: readonlyrest - allowed_protocols: [TLSv1.2] - allowed_ciphers: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] - - response_if_req_forbidden: Forbidden by Readonly Rest! - - access_control_rules: - - name: Accept all requests from localhost - type: allow - hosts: [127.0.0.1] - verbosity: error - - - name: workaround - type: allow - actions: - - "indices:data/read/field_caps*" - - "indices:admin/template/put" - verbosity: info - - - name: "::FLUENTD::" - type: allow - # auth_key is good for testing, but replace it with `auth_key_sha256`! - # auth_key: fluentd:fluentd - auth_key_sha256: 9545566c208f39b107b456430dd3b4b5b08eaa6c2abc62b2d6bcbb79c95b619c - actions: - - "cluster:monitor/main" - - "indices:admin/types/exists" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/template/*" - - "indices:admin/create" - indices: - - "agent-*" - - "apache-*" - - "bit-*" - - "fluent-*" - - "gem-*" - - "k8s-*" - - "kitchen-*" - - "mysql-*" - - "redis-*" - - "unmatched-*" - verbosity: error - - # We trust Kibana's server side process, full access granted via HTTP authentication - - name: "::KIBANA-ADMIN::" - type: allow - # auth_key is good for testing, but replace it with `auth_key_sha256`! - # auth_key: kibana:kibana - auth_key_sha256: ab8aa94dd63debfa31ef8a9eae9582dcb252c06cdb6313e123546cc8edfeaf3e - hosts: [kibana] - kibana_access: admin - verbosity: error # don't log successful request - - - name: "::KIBANA-RO::" - type: allow - # auth_key: ro:password - auth_key_sha256: 1f2c06fd49c4c8912253bcb0671f3279142c7a1d9f59bdf76a10534740332deb - hosts: [kibana] - kibana_access: ro - verbosity: error - - - name: "::KIBANA-RW::" - type: allow - # auth_key: rw:password - auth_key_sha256: 00045d3d78f2fc23914016fb8234b94f3d99e488f75c41740ac562a22fe97bc1 - hosts: [kibana] - kibana_access: rw - verbosity: error - - - name: "::ELASTICHQ::" - type: allow - # auth_key is good for testing, but replace it with `auth_key_sha256`! - # auth_key: elastichq:elastichq - auth_key_sha256: ccdff8600d84f900fe3419c286524788f8102581a40e591cd765ac724634bf15 - actions: - - "cluster:monitor/*" - - "indices:monitor/*" - verbosity: error - - - name: "::TERMINAL::" - type: allow - # auth_key is good for testing, but replace it with `auth_key_sha256`! - # auth_key: terminal:password - auth_key_sha256: fc8d9571165e073ac292f6b42b2ff9d36b80a19e6396e1de30b0a83881cd4b2a - actions: - - "cluster:monitor/*" - verbosity: error diff --git a/elasticsearch/config/warm/elasticsearch.yml b/elasticsearch/config/warm.yml similarity index 100% rename from elasticsearch/config/warm/elasticsearch.yml rename to elasticsearch/config/warm.yml diff --git a/elasticsearch/plugins/readonlyrest-1.16.21_es6.2.4.zip b/elasticsearch/plugins/readonlyrest-1.16.21_es6.2.4.zip deleted file mode 100644 index 4ab9eb0..0000000 Binary files a/elasticsearch/plugins/readonlyrest-1.16.21_es6.2.4.zip and /dev/null differ diff --git a/elasticsearch/plugins/readonlyrest-1.19.5_es7.7.1.zip b/elasticsearch/plugins/readonlyrest-1.19.5_es7.7.1.zip deleted file mode 100644 index 2f07f22..0000000 Binary files a/elasticsearch/plugins/readonlyrest-1.19.5_es7.7.1.zip and /dev/null differ diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 1139ef9..ee083f9 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -4,3 +4,11 @@ FROM docker.elastic.co/kibana/kibana:${VERSION_EFK} LABEL Author="Lee Myring " LABEL Description="Kibana instance" LABEL Version="2.0.0" + +USER root +RUN buildDeps="sudo nc" \ + && yum -y update \ + && yum -y install ${buildDeps} + +USER 1000 +HEALTHCHECK --interval=5s --timeout=2s --retries=15 CMD nc -z kibana 5601 || exit 1 diff --git a/scripts/down-es-cluster.sh b/scripts/down-es-cluster.sh new file mode 100755 index 0000000..148ebcf --- /dev/null +++ b/scripts/down-es-cluster.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +docker-compose -f docker-compose.yml down +docker-compose -f docker-compose-clients.yml down --remove-orphans +minikube delete diff --git a/scripts/helpers/common.sh b/scripts/helpers/common.sh index 600a8f7..6302782 100755 --- a/scripts/helpers/common.sh +++ b/scripts/helpers/common.sh @@ -5,11 +5,11 @@ green=`tput setaf 2` yellow=`tput setaf 3` reset=`tput sgr0` -kibanaCurlPrefix="curl -u kibana:kibana" -kibanaUrl="http://localhost:5601" +kibanaCurlPrefix="curl -k -u kibana:${ELASTICSEARCH_KIBANA_PASSWORD}" +kibanaUrl="https://kibana:5601" -elasticsearchCurlPrefix="curl -u elastichq:elastichq" -elasticsearchUrl="http://localhost:9200" +elasticsearchCurlPrefix="curl -u elastichq:${ELASTICSEARCH_ELASTIC_PASSWORD}" +elasticsearchUrl="https://localhost:9200" # Check if all the essential tools are installed function checkPreRequisites { diff --git a/scripts/start-clients.sh b/scripts/start-clients.sh new file mode 100755 index 0000000..d289856 --- /dev/null +++ b/scripts/start-clients.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +directory=$(cd `dirname $0` && pwd) +source ${directory}/helpers/common.sh +source ${directory}/../.envrc + +checkPreRequisites + +docker-compose -f docker-compose-clients.yml up -d --build + +while ! nc -z localhost 5601