From 07b5ba5bfc5609e06b644fe01d31170a6fffc3cc Mon Sep 17 00:00:00 2001 From: olivier de Meringo Date: Fri, 1 Mar 2024 13:03:00 +0100 Subject: [PATCH 1/2] doc: document adding new regions to the prometheus configuration and dashboard. --- dashboard-config/prometheus/prometheus.yml | 28 +++++++++- docs/src/SUMMARY.md | 1 + docs/src/how-to/set-up-dashboard.md | 52 +++++++++++++++++-- docs/src/reference/common-issues-and-FAQ.md | 16 ++++++ docs/src/reference/output-data.md | 2 +- .../tutorials/quickstart-dashboard-docker.md | 10 +++- 6 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 docs/src/reference/common-issues-and-FAQ.md diff --git a/dashboard-config/prometheus/prometheus.yml b/dashboard-config/prometheus/prometheus.yml index ffb7a9d3..ceb24590 100644 --- a/dashboard-config/prometheus/prometheus.yml +++ b/dashboard-config/prometheus/prometheus.yml @@ -1,6 +1,6 @@ global: - scrape_interval: 30s # By default, scrape targets every 15 seconds. - evaluation_interval: 30s # Evaluate rules every 15 seconds. + scrape_interval: 30s # By default, scrape targets every 30 seconds. + evaluation_interval: 30s # Evaluate rules every 30 seconds. # Attach these extra labels to all timeseries collected by this Prometheus instance. external_labels: @@ -44,6 +44,30 @@ scrape_configs: aws_region: ['eu-central-1'] include_block_storage: ['true'] + - job_name: 'cloud-scanner-metrics-us-east-1' + scheme: http + static_configs: + - targets: ['cloud_scanner_boa:8000'] + labels: + group: 'production' + metrics_path: '/metrics' + params: + aws_region: ['us-east-1'] + include_block_storage: ['true'] + + # Example of adding an additional scrape job to support us-east-2 region + # + # - job_name: 'cloud-scanner-metrics-us-east-2' + # scheme: http + # static_configs: + # - targets: ['cloud_scanner_boa:8000'] + # labels: + # group: 'production' + # metrics_path: '/metrics' + # params: + # aws_region: ['us-east-2'] + # include_block_storage: ['true'] + # Optional: push metics to a rem0ote prometheus instance (SAAS prometheus / grafana) # You get username and password from your prometheus SAAS dashboard. #remote_write: diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index e476bf32..f444063b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -24,6 +24,7 @@ # Reference +- [💡 Common issues and FAQ](reference/common-issues-and-FAQ.md) - [CLI options](reference/cli-options.md) - [Environment variables](reference/env-vars.md) - [OpenAPI specification in server mode](reference/openapi-server-mode.md) diff --git a/docs/src/how-to/set-up-dashboard.md b/docs/src/how-to/set-up-dashboard.md index 1f698d8a..ecbfa09d 100644 --- a/docs/src/how-to/set-up-dashboard.md +++ b/docs/src/how-to/set-up-dashboard.md @@ -9,18 +9,60 @@ The metrics are exposed in Prometheus/OpenMetrics format. You can use it to feed ## Overview 1. Start cloud-scanner in metrics mode (using the `--serve` option or by deploying it as a serverless application). -2. Setup Prometheus to scrape the metrics +2. Setup Prometheus to scrape the metrics. Metrics are generated separately for each region. 3. Configure a dashboard to display the results. ![components of monitoring stack in docker compose](../images/cloud-scanner-metrics-compose.excalidraw.png) ## Detailed steps +### Demo for some EU regions. + You can refer to the provided [docker compose example ](../tutorials/quickstart-dashboard-docker.md) for a quick start. -Prometheus and Grafana config files are in the [dashboard-config](https://github.com/Boavizta/cloud-scanner/tree/main/dashboard-config) directory. +It comes preconfigured to generate metrics for the following regions: +- eu-west-1 +- eu-west-3 +- eu-central-1 +- us-east-1 + +### Generating metrics for additional regions. + +Examples of Prometheus and Grafana configuration files are located in the [dashboard-config](https://github.com/Boavizta/cloud-scanner/tree/main/dashboard-config) directories. + +To generate (and display) metrics for an additional region: + +1. Edit the prometheus configuration (`prometheus.yml`) +2. Add a new job and set its region parameter like `aws_region: ['us-east-2']`. +3. Restart Prometheus container and verify that metrics are now collected for this region. + +In the exemple compose stack, prometheus admin GUI is exposed. You can check status of the scrape targets at: . + +```yml +# Extract from prometheus.yml +# Example extra job configuration to cover us-east-2 + - job_name: 'cloud-scanner-metrics-us-east-2' + scheme: http + static_configs: + - targets: ['cloud_scanner_boa:8000'] + labels: + group: 'production' + metrics_path: '/metrics' + params: + aws_region: ['us-east-2'] + include_block_storage: ['true'] +``` + +### Adapting configuration for production use: + +In production environment, you may want to increase the metrics scraping interval. + +This It is set to 30 seconds in the sample prometheus configuration. In production you shoud increase this intervall (e.g. 10 minutes). -For production use: -- It is easier (and safer) to deploy cloud-scanner as a serverless application. See [Deploy cloud scanner as a serverless application](deploy-sls.md). -- In production environment, you may want to increase the metrics scraping interval (30 seconds in this demo) in the prometheus configuration file. +```yml +# Prometheus scrapping interval / extract from prometheus.yml configuration +global: + scrape_interval: 30s # By default, scrape targets every 30 seconds. + evaluation_interval: 30s # Evaluate rules every 30 seconds. +``` diff --git a/docs/src/reference/common-issues-and-FAQ.md b/docs/src/reference/common-issues-and-FAQ.md new file mode 100644 index 00000000..34d36e81 --- /dev/null +++ b/docs/src/reference/common-issues-and-FAQ.md @@ -0,0 +1,16 @@ +# Common issues and FAQ + +## The demo dashboard does not show any metric... + +Several reasons may explain why metrics do not appear in the dashboard. + +1. Verify in the docker-compose logs that there is no error related to AWS authentication. +2. Check that prometheus is configured to retrieve metrics for the region you use. + +⚠ The demo docker-compose is preconfigured to generate metrics only for the following regions: +- eu-west-1 +- eu-west-3 +- eu-central-1 +- us-east-1 + +Refer to [Generating metrics for additional regions](../how-to/set-up-dashboard.md#generating-metrics-for-additional-regions) to retrieve metrics for additional regions. diff --git a/docs/src/reference/output-data.md b/docs/src/reference/output-data.md index b75879e2..8d9f99e1 100644 --- a/docs/src/reference/output-data.md +++ b/docs/src/reference/output-data.md @@ -79,7 +79,7 @@ See [OpenAPI specification in server mode](./openapi-server-mode.md) As CLI application, If using `--as-metrics` or `-m` option or the `serve` command, cloud-scanner returns consolidated results as OpenMetric/Prometheus format instead of json details. This is also the default format of the serverless app `metrics` route. -doWhen using the metric output format, you get 2 sets of metrics +When using the metric output format, you get 2 sets of metrics - Metrics named: _boavizta_xxxxx_ are _summary_ metrics (total number of resources, summed impacts, a.s.o) - Metrics named _boavizta_resource_yyy_ are specific to individual resources. The metric label can be filtered to identify resource. diff --git a/docs/src/tutorials/quickstart-dashboard-docker.md b/docs/src/tutorials/quickstart-dashboard-docker.md index 132fa0bf..706f27a1 100644 --- a/docs/src/tutorials/quickstart-dashboard-docker.md +++ b/docs/src/tutorials/quickstart-dashboard-docker.md @@ -4,7 +4,7 @@ Visualize the live impacts of you account in a dashboard. No installation needed, you will run a public docker image of cloud-scanner CLI, Boavizta API, Prometheus and Grafana to get access to a demo dashboard.. -All data remain will local to your environment (this docker-compose stack uses a _private instance_ of Boavizta API). +All data remain local to your environment (this docker-compose stack uses a _private instance_ of Boavizta API). ## Pre-requisites @@ -28,6 +28,14 @@ docker-compose up ![Demo dashboard exposing cloud scanner metrics](../images/CS-dashboard.png "A example dashboard rendering cloud scanner metrics") +⚠ The demo docker-compose is preconfigured to generate metrics only for the following regions: +- eu-west-1 +- eu-west-3 +- eu-central-1 +- us-east-1 + +Refer to [Generating metrics for additional regions](../how-to/set-up-dashboard.md#generating-metrics-for-additional-regions) to retrieve metrics for additional regions. + ## Additional info - ⚠ This docker-compose example is **not** intended for production deployment, but rather for quick testing. From 6f6558e3f316c5b3aa38cd3161c08ac99f87d497 Mon Sep 17 00:00:00 2001 From: olivier de Meringo Date: Fri, 1 Mar 2024 13:12:20 +0100 Subject: [PATCH 2/2] rel: update version in docker-compose and update changelog for release 2.0.4 --- CHANGELOG.md | 23 +++++++++++++++++++---- docker-compose.yml | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 691d00cd..e3b5379f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 _This paragraph may describe WIP/unreleased features. They are merged to main branch but not tagged._ -- [Release 2.0.3 uses dev Boavizta API URL instead of stable · Issue #425 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/425) +## [2.0.4]-2024-03-01 + +### Added + - Add a Prometheus data volume in the docker-compose example [Document how to persist prometheus data when the container is recreated · Issue #434 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/434) -- [Improve doc about passing AWS credentials · Issue #77 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/77) +- Support all AWS regions [Improve region support · Issue #48 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/48) + +### Breaking change + +- Cli and Serverless parameters `--hours-use-time` were renamed into `--use-duration-hours`. Short form remain `-u`. + +### Changed + + - [Refactor code to make it more readable · Issue #209 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/209) -- /!\ **Breaking change**: Cli and Serverless parameters `--hours-use-time` were renamed into `--use-duration-hours`. Short form remain `-u`. - [Improve error message when a region is incorrect · Issue #439 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/439) -- Support all AWS regions [Improve region support · Issue #48 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/48) +- [Release 2.0.3 uses dev Boavizta API URL instead of stable · Issue #425 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/425) +- [Improve doc about passing AWS credentials · Issue #77 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/77) + +### New contributors + +- Thanks to @jnioche for his contribution to testing and documenting issues related to regions ! ## [2.0.3]-2024-01-17 diff --git a/docker-compose.yml b/docker-compose.yml index 5770d86b..c64930f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: cloud_scanner: container_name: "cloud_scanner_boa" hostname: cloud_scanner - image: ghcr.io/boavizta/cloud-scanner-cli:2.0.3 + image: ghcr.io/boavizta/cloud-scanner-cli:2.0.4 command: - -b http://boavizta_api:5000 - -vv