diff --git a/.env b/.env index 576f904e27..40c80c1d19 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -ELASTIC_VERSION=8.15.1 +ELASTIC_VERSION=7.17.24 ## Passwords for stack users # diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 97bdac6400..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Dependabot configuration -# -# For more information, please refer to: -# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates - -version: 2 - -updates: - -# Maintain dependencies for GitHub Actions -- package-ecosystem: github-actions - directory: / - schedule: - interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594eb3a490..f51c6f8d1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ main ] + branches: [ release-7.x ] pull_request: - branches: [ main ] + branches: [ release-7.x ] jobs: @@ -124,27 +124,11 @@ jobs: - name: Execute Enterprise Search test suite run: | - - # Set mandatory Elasticsearch settings - - sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml - - # Restart Elasticsearch for changes to take effect - - docker compose restart elasticsearch - - # Run Enterprise Search and execute tests - sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search .github/workflows/scripts/run-tests-enterprise-search.sh - # Revert changes to Elasticsearch configuration - - sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml - docker compose restart elasticsearch - - name: Collect troubleshooting data id: debug-data if: failure() diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2b39074798..a1eddedd41 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,12 +1,10 @@ name: Documentation on: - schedule: - - cron: '0 0 * * 0' # At 00:00 every Sunday push: - branches: [ main ] + branches: [ release-7.x ] pull_request: - branches: [ main ] + branches: [ release-7.x ] jobs: diff --git a/.github/workflows/scripts/elasticsearch-setup-passwords.exp b/.github/workflows/scripts/elasticsearch-setup-passwords.exp new file mode 100755 index 0000000000..b0cc497465 --- /dev/null +++ b/.github/workflows/scripts/elasticsearch-setup-passwords.exp @@ -0,0 +1,23 @@ +#!/usr/bin/expect -f + +# List of expected users with dummy password +set user "(elastic|apm_system|kibana_system|logstash_system|beats_system|remote_monitoring_user)" +set password "testpasswd" + +# Find elasticsearch container id +set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch] + +set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b -u http://localhost:9200" + +spawn {*}$cmd + +expect { + -re "(E|Ree)nter password for \\\[$user\\\]: " { + send "$password\r" + exp_continue + } + eof +} + +lassign [wait] pid spawnid os_error_flag value +exit $value diff --git a/.github/workflows/scripts/run-tests-core.sh b/.github/workflows/scripts/run-tests-core.sh index e19e49f187..0af6ab6e10 100755 --- a/.github/workflows/scripts/run-tests-core.sh +++ b/.github/workflows/scripts/run-tests-core.sh @@ -27,6 +27,25 @@ grouplog 'Wait for readiness of Kibana' poll_ready "$cid_kb" 'http://kibana:5601/api/status' --resolve "kibana:5601:${ip_kb}" -u 'kibana_system:testpasswd' endgroup +log 'Creating Logstash index pattern in Kibana' +source .env +curl -X POST -D- "http://${ip_kb}:5601/api/saved_objects/index-pattern" \ + -s -w '\n' \ + -H 'Content-Type: application/json' \ + -H "kbn-version: ${ELASTIC_VERSION}" \ + -u elastic:testpasswd \ + -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' + +log 'Searching index pattern via Kibana API' +response="$(curl "http://${ip_kb}:5601/api/saved_objects/_find?type=index-pattern" -s -u elastic:testpasswd)" +echo "$response" +declare -i count +count="$(jq -rn --argjson data "${response}" '$data.total')" +if (( count != 1 )); then + echo "Expected 1 index pattern, got ${count}" + exit 1 +fi + log 'Sending message to Logstash TCP input' declare -i was_retried=0 @@ -50,7 +69,7 @@ fi # need to be resilient here. was_retried=0 declare -a refresh_args=( '-X' 'POST' '-s' '-w' '%{http_code}' '-u' 'elastic:testpasswd' - 'http://elasticsearch:9200/logs-generic-default/_refresh' + 'http://elasticsearch:9200/logstash-*/_refresh' '--resolve' "elasticsearch:9200:${ip_es}" ) @@ -76,7 +95,7 @@ log 'Searching message in Elasticsearch' # we need to be resilient here too. was_retried=0 declare -a search_args=( '-s' '-u' 'elastic:testpasswd' - 'http://elasticsearch:9200/logs-generic-default/_search?q=message:dockerelk&pretty' + 'http://elasticsearch:9200/logstash-*/_search?q=message:dockerelk&pretty' '--resolve' "elasticsearch:9200:${ip_es}" ) declare -i count diff --git a/.github/workflows/scripts/run-tests-fleet.sh b/.github/workflows/scripts/run-tests-fleet.sh index ee07e5f7e7..331f7ea9d0 100755 --- a/.github/workflows/scripts/run-tests-fleet.sh +++ b/.github/workflows/scripts/run-tests-fleet.sh @@ -70,44 +70,3 @@ if (( count == 0 )); then echo 'Expected at least 1 document' exit 1 fi - -# We expect to find log entries for the 'elasticsearch' Compose service using -# the following query: -# -# agent.name:"fleet-server" -# AND agent.type:"filebeat" -# AND container.name:"docker-elk-elasticsearch-1" -# -log 'Searching a container document generated by Fleet Server' - -response= -count=0 - -was_retried=0 - -# retry for max 60s (30*2s) -for _ in $(seq 1 30); do - response="$(curl 'http://elasticsearch:9200/logs-docker.container_logs-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22filebeat%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)" - - set +u # prevent "unbound variable" if assigned value is not an integer - count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')" - set -u - - if (( count > 0 )); then - break - fi - - was_retried=1 - echo -n 'x' >&2 - sleep 2 -done -if ((was_retried)); then - # flush stderr, important in non-interactive environments (CI) - echo >&2 -fi - -echo "$response" -if (( count == 0 )); then - echo 'Expected at least 1 document' - exit 1 -fi diff --git a/.github/workflows/spam-issue-close.yml b/.github/workflows/spam-issue-close.yml deleted file mode 100644 index 4de11c11c4..0000000000 --- a/.github/workflows/spam-issue-close.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Close issues without context - -permissions: - issues: write - -on: - issues: - types: [ labeled ] - -jobs: - - close-lock: - name: Close and lock issues - if: contains(github.event.issue.labels.*.name, 'bot:close') && github.event.issue.state == 'open' - runs-on: ubuntu-latest - - steps: - - name: Close - id: close - uses: actions/stale@v9.0.0 - with: - days-before-issue-stale: -1 - days-before-issue-close: 0 - stale-issue-label: bot:close - close-issue-label: insufficient information - close-issue-message: >- - This description omits all, or critical parts of the information requested by maintainers to be able to - reproduce the issue: - - - - the **complete** log history of your Elastic components, including `setup`. - - any change(s) performed to the docker-elk configuration. - - details about the runtime environment, for both Docker and Compose. - - - Therefore, this issue will now be **closed**. Please open a new issue and fill in the template. It saves - everyone's efforts, and allows maintainers to provide you with a solution in as few round trips as possible. - - Thank you for your understanding. :pray: - - # Due to eventual consistency, listing closed issues immediately after a - # close does not always yield the expected results. A sleep is a simple - # enough remediation to this issue. - - name: Pause - if: fromJson(steps.close.outputs.closed-issues-prs)[0] - run: sleep 5 - - - name: Lock - uses: dessant/lock-threads@v5 - if: fromJson(steps.close.outputs.closed-issues-prs)[0] - with: - process-only: issues - issue-inactive-days: 0 - include-any-issue-labels: bot:close - remove-issue-labels: bot:close - issue-lock-reason: spam - log-output: true diff --git a/.github/workflows/update-merge.yml b/.github/workflows/update-merge.yml deleted file mode 100644 index b7670b1fe0..0000000000 --- a/.github/workflows/update-merge.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Merge Elastic updates - -on: - workflow_run: - workflows: [ CI ] - types: - - completed - branches: - - update/main - - update/tls - - update/release-7.x - -jobs: - - merge: - name: Merge pull request - if: github.event.workflow_run.conclusion == 'success' - runs-on: ubuntu-latest - - steps: - - name: Impersonate update bot - uses: actions/create-github-app-token@v1 - id: generate-token - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - - - name: Approve and merge - uses: ridedott/merge-me-action@v2 - with: - GITHUB_LOGIN: docker-elk-updater - GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} - - - name: Delete branch - uses: actions/github-script@v7 - with: - script: | - await github.request('DELETE /repos/{owner}/{repo}/git/refs/{ref}', { - owner: '${{ github.event.workflow_run.repository.owner.login }}', - repo: '${{ github.event.workflow_run.repository.name }}', - ref: 'heads/${{ github.event.workflow_run.head_branch }}' - }) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml deleted file mode 100644 index 09752dae55..0000000000 --- a/.github/workflows/update.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Update Elastic release - -on: - schedule: - - cron: '0 0 * * 0' # At 00:00 every Sunday - -jobs: - - check-and-update: - name: Check and update Elastic release - runs-on: ubuntu-latest - strategy: - matrix: - include: - - release: 8.x - branch: main - - release: 8.x - branch: tls - - release: 7.x - branch: release-7.x - - steps: - - uses: actions/setup-node@v4 - - run: npm install semver - - - name: Get latest release version - uses: actions/github-script@v7 - id: get-latest-release - with: - script: | - const semver = require('semver') - - const latestVersion = await github. - paginate(github.rest.repos.listReleases, { - owner: 'elastic', - repo: 'elasticsearch' - }) - .then(releases => { - for (const release of releases) { - // Results are returned sorted by created_at, so it is safe to assume - // that the first encountered match is also the series' latest release. - - const version=semver.clean(release.tag_name) - - if (semver.satisfies(version, '${{ matrix.release }}')) { - return version - } - } - }); - - if (latestVersion) { - // Return an object so that the result can be handled as structured data - // instead of a quoted string in subsequent steps. - return { version: latestVersion } - } - - - uses: actions/checkout@v4 - if: steps.get-latest-release.outputs.result - with: - ref: ${{ matrix.branch }} - - - name: Update stack version - id: update-files - if: steps.get-latest-release.outputs.result - run: | - source .env - cur_ver="$ELASTIC_VERSION" - new_ver=${{ fromJson(steps.get-latest-release.outputs.result).version }} - - # Escape period characters so sed interprets them literally - cur_ver="${cur_ver//./\\.}" - - declare -a upd_files=( .env README.md */Dockerfile extensions/*/Dockerfile ) - if [ -f tls/README.md ]; then - upd_files+=( tls/README.md ) - fi - - sed -i "s/${cur_ver}/${new_ver}/g" "${upd_files[@]}" - - git_status="$(git status --porcelain)" - if [[ ${git_status} ]]; then - echo -e 'Changes to be committed:\n' - echo "${git_status}" - echo 'has-changes=true' >>"$GITHUB_OUTPUT" - fi - - - name: Impersonate update bot - uses: actions/create-github-app-token@v1 - id: generate-token - if: steps.update-files.outputs.has-changes - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - - - name: Send pull request to update to new version - if: steps.update-files.outputs.has-changes - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ steps.generate-token.outputs.token }} - branch: update/${{ matrix.branch }} - commit-message: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} - title: Update to v${{ fromJson(steps.get-latest-release.outputs.result).version }} - delete-branch: true diff --git a/README.md b/README.md index ee046c2cb0..5109886608 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ # Elastic stack (ELK) on Docker -[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.15.1-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) -[![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Amain) +[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-7.17.24-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases) +[![Build Status](https://github.com/deviantony/docker-elk/workflows/CI/badge.svg?branch=release-7.x)](https://github.com/deviantony/docker-elk/actions?query=workflow%3ACI+branch%3Arelease-7.x) [![Join the chat](https://badges.gitter.im/Join%20Chat.svg)](https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im) -Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose. +Run the version 7.x of the [Elastic stack][elk-stack] with Docker and Docker Compose. It gives you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch and the visualization power of Kibana. Based on the [official Docker images][elastic-docker] from Elastic: -* [Elasticsearch](https://github.com/elastic/elasticsearch/tree/main/distribution/docker) -* [Logstash](https://github.com/elastic/logstash/tree/main/docker) -* [Kibana](https://github.com/elastic/kibana/tree/main/src/dev/build/tasks/os_packages/docker_generator) +* [Elasticsearch](https://github.com/elastic/elasticsearch/tree/7.17/distribution/docker) +* [Logstash](https://github.com/elastic/logstash/tree/7.17/docker) +* [Kibana](https://github.com/elastic/kibana/tree/7.17/src/dev/build/tasks/os_packages/docker_generator) Other available stack variants: @@ -67,6 +67,7 @@ own_. [sherifabdlnaby/elastdocker][elastdocker] is one example among others of p * [Initial setup](#initial-setup) * [Setting up user authentication](#setting-up-user-authentication) * [Injecting data](#injecting-data) + * [Default Kibana index pattern creation](#default-kibana-index-pattern-creation) * [Cleanup](#cleanup) * [Version selection](#version-selection) 1. [Configuration](#configuration) @@ -136,7 +137,7 @@ instructions from the [documentation][mac-filesharing] to add more locations. Clone this repository onto the Docker host that will run the stack with the command below: ```sh -git clone https://github.com/deviantony/docker-elk.git +git clone --branch release-7.x https://github.com/deviantony/docker-elk.git ``` Then, initialize the Elasticsearch users and groups required by docker-elk by executing the command: @@ -174,32 +175,20 @@ browser and use the following (default) credentials to log in: > [!NOTE] > Refer to [Security settings in Elasticsearch][es-security] to disable authentication. -> [!WARNING] -> Starting with Elastic v8.0.0, it is no longer possible to run Kibana using the bootstraped privileged `elastic` user. - The _"changeme"_ password set by default for all aforementioned users is **unsecure**. For increased security, we will reset the passwords of all aforementioned Elasticsearch users to random secrets. -1. Reset passwords for default users +1. Reset passwords for built-in users - The commands below reset the passwords of the `elastic`, `logstash_internal` and `kibana_system` users. Take note - of them. + The command below resets the passwords of all 6 [built-in users][builtin-users]. Take note of them. ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user elastic + docker compose exec -T elasticsearch bin/elasticsearch-setup-passwords auto --batch ``` - ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user logstash_internal - ``` - - ```sh - docker compose exec elasticsearch bin/elasticsearch-reset-password --batch --user kibana_system - ``` - - If the need for it arises (e.g. if you want to [collect monitoring information][ls-monitoring] through Beats and - other components), feel free to repeat this operation at any time for the rest of the [built-in - users][builtin-users]. + The password of the `logstash_internal` user must be changed manually, either from Kibana's [Stack Management > + Users][kibana-sec] view, or using the procedure described at [How to reset a password + programmatically](#how-to-reset-a-password-programmatically). 1. Replace usernames and passwords in configuration files @@ -253,6 +242,37 @@ cat /path/to/logfile.log | nc --send-only localhost 50000 # nmap You can also load the sample data provided by your Kibana installation. +#### Default Kibana index pattern creation + +When Kibana launches for the first time, it is not configured with any index pattern. + +##### Via the Kibana web UI + +> [!NOTE] +> You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web UI. + +Navigate to the _Discover_ view of Kibana from the left sidebar. You will be prompted to create an index pattern. Enter +`logstash-*` to match Logstash indices then, on the next page, select `@timestamp` as the time filter field. Finally, +click _Create index pattern_ and return to the _Discover_ view to inspect your log entries. + +Refer to [Connect Kibana with Elasticsearch][connect-kibana] and [Creating an index pattern][index-pattern] for detailed +instructions about the index pattern configuration. + +##### On the command line + +Create an index pattern via the Kibana API: + +```sh +curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ + -H 'Content-Type: application/json' \ + -H 'kbn-version: 7.17.24' \ + -u elastic: \ + -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' +``` + +The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the +first time. + ### Cleanup Elasticsearch data is persisted inside a volume by default. @@ -265,9 +285,6 @@ docker compose down -v ### Version selection -This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major -version (8.x). - To use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env) file. If you are upgrading an existing stack, remember to rebuild all container images using the `docker compose build` command. @@ -276,12 +293,6 @@ command. > Always pay attention to the [official upgrade instructions][upgrade] for each individual component before performing a > stack upgrade. -Older major versions are also supported on separate branches: - -* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series -* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life) -* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life) - ## Configuration > [!IMPORTANT] @@ -462,10 +473,10 @@ See the following Wiki pages: [elk-stack]: https://www.elastic.co/what-is/elk-stack [elastic-docker]: https://www.docker.elastic.co/ [subscriptions]: https://www.elastic.co/subscriptions -[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html -[license-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/license-settings.html -[license-mngmt]: https://www.elastic.co/guide/en/kibana/current/managing-licenses.html -[license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/current/licensing-apis.html +[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-settings.html +[license-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/license-settings.html +[license-mngmt]: https://www.elastic.co/guide/en/kibana/7.17/managing-licenses.html +[license-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/licensing-apis.html [elastdocker]: https://github.com/sherifabdlnaby/elastdocker @@ -473,26 +484,26 @@ See the following Wiki pages: [compose-install]: https://docs.docker.com/compose/install/ [linux-postinstall]: https://docs.docker.com/engine/install/linux-postinstall/ -[bootstrap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html -[es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/current/system-config.html -[es-heap]: https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings +[bootstap-checks]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/bootstrap-checks.html +[es-sys-config]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/system-config.html +[es-heap]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/important-settings.html#heap-size-settings [win-filesharing]: https://docs.docker.com/desktop/settings/windows/#file-sharing [mac-filesharing]: https://docs.docker.com/desktop/settings/mac/#file-sharing -[builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html -[ls-monitoring]: https://www.elastic.co/guide/en/logstash/current/monitoring-with-metricbeat.html -[sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/current/secure-cluster.html +[builtin-users]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/built-in-users.html +[sec-cluster]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/secure-cluster.html +[kibana-sec]: https://www.elastic.co/guide/en/kibana/7.17/xpack-security.html -[connect-kibana]: https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html -[index-pattern]: https://www.elastic.co/guide/en/kibana/current/index-patterns.html +[connect-kibana]: https://www.elastic.co/guide/en/kibana/7.17/connect-to-elasticsearch.html +[index-pattern]: https://www.elastic.co/guide/en/kibana/7.17/index-patterns.html [config-es]: ./elasticsearch/config/elasticsearch.yml [config-kbn]: ./kibana/config/kibana.yml [config-ls]: ./logstash/config/logstash.yml -[es-docker]: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html -[kbn-docker]: https://www.elastic.co/guide/en/kibana/current/docker.html -[ls-docker]: https://www.elastic.co/guide/en/logstash/current/docker-config.html +[es-docker]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docker.html +[kbn-docker]: https://www.elastic.co/guide/en/kibana/7.17/docker.html +[ls-docker]: https://www.elastic.co/guide/en/logstash/7.17/docker-config.html -[upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html +[upgrade]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/setup-upgrade.html diff --git a/docker-compose.yml b/docker-compose.yml index e7c07b992f..02e00d8486 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,7 +59,7 @@ services: # Elasticsearch. Ignored on subsequent runs. ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} # Use single node discovery in order to disable production mode and avoid bootstrap checks. - # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html + # see: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/bootstrap-checks.html discovery.type: single-node networks: - elk diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 39e4ab3c36..7a2c35ce6e 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-7.17.24} # Add your elasticsearch plugins setup here # Example: RUN elasticsearch-plugin install analysis-icu diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml index d66f071aad..c6a62cf2eb 100644 --- a/elasticsearch/config/elasticsearch.yml +++ b/elasticsearch/config/elasticsearch.yml @@ -1,12 +1,15 @@ --- ## Default Elasticsearch configuration from Elasticsearch base image. -## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml +## https://github.com/elastic/elasticsearch/blob/7.17/distribution/docker/src/docker/config/elasticsearch.yml # cluster.name: docker-cluster network.host: 0.0.0.0 ## X-Pack settings -## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html +## see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-settings.html # xpack.license.self_generated.type: trial xpack.security.enabled: true +# The API key service is required by Fleet, but disabled by default when the +# Elasticsearch API is not served over TLS. +xpack.security.authc.api_key.enabled: true diff --git a/extensions/curator/Dockerfile b/extensions/curator/Dockerfile index 1836c7f7f4..9fc4f9acbe 100644 --- a/extensions/curator/Dockerfile +++ b/extensions/curator/Dockerfile @@ -1,4 +1,4 @@ -FROM untergeek/curator:8.0.10 +FROM untergeek/curator:7.0.1 USER root diff --git a/extensions/curator/README.md b/extensions/curator/README.md index e19ebe7039..fb8ac1ff04 100644 --- a/extensions/curator/README.md +++ b/extensions/curator/README.md @@ -17,4 +17,4 @@ All configuration files are available in the `config/` directory. ## Documentation -[Curator Reference](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html) +[Curator Reference](https://www.elastic.co/guide/en/elasticsearch/client/curator/7.0/index.html) diff --git a/extensions/curator/config/curator.yml b/extensions/curator/config/curator.yml index 6777edc9cb..1887f65da1 100644 --- a/extensions/curator/config/curator.yml +++ b/extensions/curator/config/curator.yml @@ -1,5 +1,5 @@ # Curator configuration -# https://www.elastic.co/guide/en/elasticsearch/client/curator/current/configfile.html +# https://www.elastic.co/guide/en/elasticsearch/client/curator/7.0/configfile.html elasticsearch: client: diff --git a/extensions/enterprise-search/Dockerfile b/extensions/enterprise-search/Dockerfile index a68c519336..cde22cb8ce 100644 --- a/extensions/enterprise-search/Dockerfile +++ b/extensions/enterprise-search/Dockerfile @@ -1,4 +1,4 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION:-7.17.24} diff --git a/extensions/enterprise-search/README.md b/extensions/enterprise-search/README.md index e8ac5735ee..6561b8096d 100644 --- a/extensions/enterprise-search/README.md +++ b/extensions/enterprise-search/README.md @@ -55,7 +55,7 @@ xpack.security.authc.api_key.enabled: true ### Configure the Enterprise Search host in Kibana -Kibana acts as the [management interface][enterprisesearch-kb] to Enterprise Search. +Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search. To enable the management experience for Enterprise Search, modify the Kibana configuration file in [`kibana/config/kibana.yml`][config-kbn] and add the following setting: @@ -132,13 +132,13 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock [config-enterprisesearch]: ./config/enterprise-search.yml -[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/current/encryption-keys.html -[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/current/workplace-search-security.html -[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html -[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html -[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html -[enterprisesearch-kb]: https://www.elastic.co/guide/en/kibana/current/enterprise-search-settings-kb.html +[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/7.17/encryption-keys.html +[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/7.17/workplace-search-security.html +[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/7.17/configuration.html +[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/7.17/docker.html +[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/7.17/index.html +[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/7.17/user-interfaces.html -[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings +[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-settings.html#api-key-service-settings [config-es]: ../../elasticsearch/config/elasticsearch.yml [config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index a1f098dd2e..c6193c5ab5 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -1,6 +1,6 @@ --- ## Enterprise Search core configuration -## https://www.elastic.co/guide/en/enterprise-search/current/configuration.html +## https://www.elastic.co/guide/en/enterprise-search/7.17/configuration.html # ## --------------------- REQUIRED --------------------- diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile index d1001bdb2d..0aeec7f237 100644 --- a/extensions/filebeat/Dockerfile +++ b/extensions/filebeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-7.17.24} diff --git a/extensions/filebeat/README.md b/extensions/filebeat/README.md index b1fcb094e3..ac73051405 100644 --- a/extensions/filebeat/README.md +++ b/extensions/filebeat/README.md @@ -35,8 +35,8 @@ container: [Run Filebeat on Docker][filebeat-docker]. [Filebeat documentation][filebeat-doc] -[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html -[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html -[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html +[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-reference-yml.html +[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/7.17/running-on-docker.html +[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/7.17/index.html [setup]: ../../README.md#how-to-re-execute-the-setup diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml index 119d5d5b7f..dee2b22059 100644 --- a/extensions/filebeat/config/filebeat.yml +++ b/extensions/filebeat/config/filebeat.yml @@ -1,5 +1,5 @@ ## Filebeat configuration -## https://github.com/elastic/beats/blob/main/deploy/docker/filebeat.docker.yml +## https://github.com/elastic/beats/blob/7.17/deploy/docker/filebeat.docker.yml # name: filebeat @@ -46,7 +46,7 @@ output.elasticsearch: password: ${FILEBEAT_INTERNAL_PASSWORD} ## HTTP endpoint for health checking -## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html +## https://www.elastic.co/guide/en/beats/filebeat/7.17/http-endpoint.html # http: diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml index 595a7d455e..60bd44dc5a 100644 --- a/extensions/filebeat/filebeat-compose.yml +++ b/extensions/filebeat/filebeat-compose.yml @@ -12,7 +12,7 @@ services: - -e # Disable config file permissions checks. Allows mounting # 'config/filebeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # see: https://www.elastic.co/guide/en/beats/libbeat/7.17/config-file-permissions.html - --strict.perms=false volumes: - ./extensions/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z diff --git a/extensions/fleet/Dockerfile b/extensions/fleet/Dockerfile index acfe9da8dc..fb1e83ae4b 100644 --- a/extensions/fleet/Dockerfile +++ b/extensions/fleet/Dockerfile @@ -1,6 +1,6 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/elastic-agent:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/beats/elastic-agent:${ELASTIC_VERSION:-7.17.24} # Ensure the 'state' directory exists and is owned by the 'elastic-agent' user, # otherwise mounting a named volume in that location creates a directory owned diff --git a/extensions/fleet/README.md b/extensions/fleet/README.md index bd8efded7f..1bf4b8abb1 100644 --- a/extensions/fleet/README.md +++ b/extensions/fleet/README.md @@ -37,6 +37,9 @@ management UI: [Fleet UI Settings][fleet-cfg]. ## Known Issues +- Logs are only collected within the Fleet Server's container. We would like to emulate the behaviour of the existing + Filebeat extension, and collect logs from all ELK containers out-of-the-box. Unfortunately, this use-case is only + supported starting with the version 8.x of the Elastic stack. - The Elastic Agent auto-enrolls using the `elastic` super-user. With this approach, you do not need to generate a service token — either using the Fleet management UI or [CLI utility][es-svc-token] — prior to starting this extension. However convenient that is, this approach _does not follow security best practices_, and we recommend @@ -53,10 +56,10 @@ management UI: [Fleet UI Settings][fleet-cfg]. ![elastic-agent-dashboard](https://user-images.githubusercontent.com/3299086/202701404-958f8d80-a7a0-4044-bbf9-bf73f3bdd17a.png "Elastic Agent Dashboard") -[fleet-doc]: https://www.elastic.co/guide/en/fleet/current/fleet-overview.html -[fleet-pol]: https://www.elastic.co/guide/en/fleet/current/agent-policy.html -[fleet-cfg]: https://www.elastic.co/guide/en/fleet/current/fleet-settings.html +[fleet-doc]: https://www.elastic.co/guide/en/fleet/7.17/fleet-overview.html +[fleet-pol]: https://www.elastic.co/guide/en/fleet/7.17/agent-policy.html +[fleet-cfg]: https://www.elastic.co/guide/en/fleet/7.17/fleet-settings.html [config-kbn]: ../../kibana/config/kibana.yml -[es-svc-token]: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-tokens-command.html +[es-svc-token]: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/service-tokens-command.html diff --git a/extensions/fleet/fleet-compose.yml b/extensions/fleet/fleet-compose.yml index a81d33bb52..39f6fa5f1a 100644 --- a/extensions/fleet/fleet-compose.yml +++ b/extensions/fleet/fleet-compose.yml @@ -4,15 +4,10 @@ services: context: extensions/fleet/ args: ELASTIC_VERSION: ${ELASTIC_VERSION} - # Run as 'root' instead of 'elastic-agent' (uid 1000) to allow reading - # 'docker.sock' and the host's filesystem. + # Run as 'root' instead of 'elastic-agent' (uid 1000) to allow reading 'docker.sock'. user: root volumes: - fleet-server:/usr/share/elastic-agent/state:Z - - type: bind - source: /var/lib/docker/containers - target: /var/lib/docker/containers - read_only: true - type: bind source: /var/run/docker.sock target: /var/run/docker.sock diff --git a/extensions/heartbeat/Dockerfile b/extensions/heartbeat/Dockerfile index 7731d66d6b..cce02a9094 100644 --- a/extensions/heartbeat/Dockerfile +++ b/extensions/heartbeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-7.17.24} diff --git a/extensions/heartbeat/README.md b/extensions/heartbeat/README.md index 64a761bb81..d14c1b1de2 100644 --- a/extensions/heartbeat/README.md +++ b/extensions/heartbeat/README.md @@ -34,8 +34,8 @@ Docker container: [Run Heartbeat on Docker][heartbeat-docker]. [Heartbeat documentation][heartbeat-doc] -[heartbeat-config]: https://www.elastic.co/guide/en/beats/heartbeat/current/heartbeat-reference-yml.html -[heartbeat-docker]: https://www.elastic.co/guide/en/beats/heartbeat/current/running-on-docker.html -[heartbeat-doc]: https://www.elastic.co/guide/en/beats/heartbeat/current/index.html +[heartbeat-config]: https://www.elastic.co/guide/en/beats/heartbeat/7.17/heartbeat-reference-yml.html +[heartbeat-docker]: https://www.elastic.co/guide/en/beats/heartbeat/7.17/running-on-docker.html +[heartbeat-doc]: https://www.elastic.co/guide/en/beats/heartbeat/7.17/index.html [setup]: ../../README.md#how-to-re-execute-the-setup diff --git a/extensions/heartbeat/config/heartbeat.yml b/extensions/heartbeat/config/heartbeat.yml index b1416ea4a9..6d60801c8b 100644 --- a/extensions/heartbeat/config/heartbeat.yml +++ b/extensions/heartbeat/config/heartbeat.yml @@ -1,5 +1,5 @@ ## Heartbeat configuration -## https://github.com/elastic/beats/blob/main/deploy/docker/heartbeat.docker.yml +## https://github.com/elastic/beats/blob/7.17/deploy/docker/heartbeat.docker.yml # name: heartbeat @@ -32,7 +32,7 @@ output.elasticsearch: password: ${HEARTBEAT_INTERNAL_PASSWORD} ## HTTP endpoint for health checking -## https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html +## https://www.elastic.co/guide/en/beats/heartbeat/7.17/http-endpoint.html # http: diff --git a/extensions/heartbeat/heartbeat-compose.yml b/extensions/heartbeat/heartbeat-compose.yml index 103d0dfae9..94b8f45524 100644 --- a/extensions/heartbeat/heartbeat-compose.yml +++ b/extensions/heartbeat/heartbeat-compose.yml @@ -9,7 +9,7 @@ services: - -e # Disable config file permissions checks. Allows mounting # 'config/heartbeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # see: https://www.elastic.co/guide/en/beats/libbeat/7.17/config-file-permissions.html - --strict.perms=false volumes: - ./extensions/heartbeat/config/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro,Z diff --git a/extensions/metricbeat/Dockerfile b/extensions/metricbeat/Dockerfile index b9313e4994..285acceb88 100644 --- a/extensions/metricbeat/Dockerfile +++ b/extensions/metricbeat/Dockerfile @@ -1,3 +1,3 @@ ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-7.17.24} diff --git a/extensions/metricbeat/README.md b/extensions/metricbeat/README.md index 2d042baa18..cf3c5119e9 100644 --- a/extensions/metricbeat/README.md +++ b/extensions/metricbeat/README.md @@ -42,8 +42,8 @@ Docker container: [Run Metricbeat on Docker][metricbeat-docker]. ![host-dashboard](https://user-images.githubusercontent.com/3299086/202710594-0deccf40-3a9a-4e63-8411-2e0d9cc6ad3a.png "Host Overview Dashboard") -[metricbeat-config]: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-reference-yml.html -[metricbeat-docker]: https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-docker.html -[metricbeat-doc]: https://www.elastic.co/guide/en/beats/metricbeat/current/index.html +[metricbeat-config]: https://www.elastic.co/guide/en/beats/metricbeat/7.17/metricbeat-reference-yml.html +[metricbeat-docker]: https://www.elastic.co/guide/en/beats/metricbeat/7.17/running-on-docker.html +[metricbeat-doc]: https://www.elastic.co/guide/en/beats/metricbeat/7.17/index.html [setup]: ../../README.md#how-to-re-execute-the-setup diff --git a/extensions/metricbeat/config/metricbeat.yml b/extensions/metricbeat/config/metricbeat.yml index 1c2b6cb87d..f6630fc007 100644 --- a/extensions/metricbeat/config/metricbeat.yml +++ b/extensions/metricbeat/config/metricbeat.yml @@ -1,5 +1,5 @@ ## Metricbeat configuration -## https://github.com/elastic/beats/blob/main/deploy/docker/metricbeat.docker.yml +## https://github.com/elastic/beats/blob/7.17/deploy/docker/metricbeat.docker.yml # name: metricbeat @@ -64,7 +64,7 @@ output.elasticsearch: password: ${METRICBEAT_INTERNAL_PASSWORD} ## HTTP endpoint for health checking -## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html +## https://www.elastic.co/guide/en/beats/metricbeat/7.17/http-endpoint.html # http: diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index e7aa67f5da..6324aacdc2 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -12,7 +12,7 @@ services: - -e # Disable config file permissions checks. Allows mounting # 'config/metricbeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # see: https://www.elastic.co/guide/en/beats/libbeat/7.17/config-file-permissions.html - --strict.perms=false # Mount point of the host’s filesystem. Required to monitor the host # from within a container. diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 7e4c501eba..2961bb1491 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,7 +1,7 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-7.17.24} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index ef3f024010..f383040402 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -1,6 +1,6 @@ --- ## Default Kibana configuration from Kibana base image. -## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts +## https://github.com/elastic/kibana/blob/7.17/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts # server.name: kibana server.host: 0.0.0.0 @@ -17,18 +17,18 @@ elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} ## Encryption keys (optional but highly recommended) ## ## Generate with either -## $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate +## $ docker container run --rm docker.elastic.co/kibana/kibana:7.17.9 bin/kibana-encryption-keys generate ## $ openssl rand -hex 32 ## -## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html -## https://www.elastic.co/guide/en/kibana/current/kibana-encryption-keys.html +## https://www.elastic.co/guide/en/kibana/7.17/using-kibana-with-security.html +## https://www.elastic.co/guide/en/kibana/7.17/kibana-encryption-keys.html # #xpack.security.encryptionKey: #xpack.encryptedSavedObjects.encryptionKey: #xpack.reporting.encryptionKey: ## Fleet -## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html +## https://www.elastic.co/guide/en/kibana/7.17/fleet-settings-kb.html # xpack.fleet.agents.fleet_server.hosts: [ http://fleet-server:8220 ] @@ -38,7 +38,6 @@ xpack.fleet.outputs: type: elasticsearch hosts: [ http://elasticsearch:9200 ] is_default: true - is_default_monitoring: true xpack.fleet.packages: - name: fleet_server diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 348ca41dbe..c90c7ad000 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,7 +1,7 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-7.17.24} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a81b89bc79..e973163868 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -1,6 +1,6 @@ --- ## Default Logstash configuration from Logstash base image. -## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml +## https://github.com/elastic/logstash/blob/7.17/docker/data/logstash/config/logstash-full.yml # http.host: 0.0.0.0 diff --git a/setup/Dockerfile b/setup/Dockerfile index 86254a138d..b060ebddb7 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,6 +1,6 @@ ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.15.1} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-7.17.24} ENTRYPOINT ["/entrypoint.sh"]