From 0b0567b650044a3c7e5facfbb6b54aefc7cb4da9 Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Fri, 8 Nov 2024 22:11:23 +0530 Subject: [PATCH 1/5] Added yaml file changes and prometheus builder changes that was removed from block-sync. Signed-off-by: Kushal Shukla --- .../3b_prometheus-test-pr_deployment.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml index 871773807..c16893a4a 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml @@ -34,7 +34,7 @@ spec: runAsUser: 0 initContainers: - name: prometheus-builder - image: docker.io/prominfra/prometheus-builder:master + image: kushalshukla/builder imagePullPolicy: Always env: - name: PR_NUMBER @@ -48,6 +48,18 @@ spec: volumeMounts: - name: prometheus-executable mountPath: /prometheus-builder + - name: key + mountPath: /config + - name: data-downloader + image: kushalshukla/writer + imagePullPolicy: Always + volumeMounts: + - name: instance-ssd + mountPath: /data + - name: bucket-config + mountPath: /config + - name: key + mountPath: /key containers: - name: prometheus image: quay.io/prometheus/busybox:latest @@ -93,6 +105,11 @@ spec: path: /mnt/disks/ssd0 - name: prometheus-executable emptyDir: {} + - name: bucket-config # Define the Secret volume + secret: + secretName: bucket-secret + - name: key + emptyDir: {} terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }} From 96e67aac67152851416b3379b0184822b147c379 Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Tue, 19 Nov 2024 15:23:08 +0530 Subject: [PATCH 2/5] Added --storage.tsdb.retention.size to retain data for certain size. Signed-off-by: Kushal Shukla --- .../prombench/benchmark/3b_prometheus-test-pr_deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml index c16893a4a..bfcc60591 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml @@ -79,7 +79,8 @@ spec: "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-pr", "--storage.tsdb.path=/prometheus", "--config.file=/etc/prometheus/prometheus.yml", - "--log.level=debug" + "--log.level=debug", + "--storage.tsdb.retention.size=5GB" ] resources: requests: From 6d462bcba7ded84fbc5091536bed965fd70e88db Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Sat, 14 Dec 2024 20:49:43 +0530 Subject: [PATCH 3/5] Adding Bucket-config in configMap instead on prometheus Repository. Signed-off-by: Kushal Shukla --- .../3b_prometheus-test-pr_deployment.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml index bfcc60591..63f348d28 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml @@ -1,3 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: + bucket-config.yml: | + path: gendata + minTime: 1729694072541 + maxTime: 1729706400000 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -110,7 +121,8 @@ spec: secret: secretName: bucket-secret - name: key - emptyDir: {} + configMap: + name: blocksync-config terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }} From 78397b0e0f8fd14c650a76608e5b52a5aa4a6576 Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Wed, 25 Dec 2024 12:42:41 +0530 Subject: [PATCH 4/5] written an instruct to create bucket-config.yml file on Readme. Signed-off-by: Kushal Shukla --- prombench/docs/eks.md | 68 ++++++++++++++++++- prombench/docs/gke.md | 67 +++++++++++++++++- prombench/docs/kind.md | 66 +++++++++++++++++- .../3b_prometheus-test-pr_deployment.yaml | 11 --- 4 files changed, 196 insertions(+), 16 deletions(-) diff --git a/prombench/docs/eks.md b/prombench/docs/eks.md index f529f57ca..2688f9ea1 100644 --- a/prombench/docs/eks.md +++ b/prombench/docs/eks.md @@ -107,8 +107,72 @@ Run Prombench tests in [Elastic Kubernetes Service (EKS)](https://aws.amazon.com ```bash make node_create ``` - -3. **Deploy the Kubernetes Objects**: +3. **Setting Up Benchmarking Data** + + When setting up a benchmarking environment, it’s often useful to have pre-generated data available. This data can help speed up testing and make benchmarks more realistic by simulating actual workloads. + +In this setup, you have two choices: + +Here’s how each option works: +- **Option 1: Download data from object storage** + + To download data from object storage, create a Kubernetes secret with exact named `bucket-secret` and file name `object-config.yml` with the necessary credentials as per your object storage. This secret enables access to the stored data. +> Note: Make sure this secret applied before `3b_prometheus-test_deployment.yaml` + +- **Option 2: Skip downloading data** + +If you don’t Want to Download data create an empty secret like this - + +```yaml +# Empty Secret to Skip Downloading Data +apiVersion: v1 +kind: Secret +metadata: + name: bucket-secret + namespace: prombench-{{ .PR_NUMBER }} +type: Opaque +stringData: + object-config.yml: +``` + +Regardless of the option chosen, data stored in Prometheus will only be retained based on the configured retention settings (```--storage.tsdb.retention.size```). + +> **⚠️ Warning:** The benchmark will change its basis when the retention size limit is reached and older downloaded blocks are deleted. Ensure that you have sufficient retention settings configured to avoid data loss that could affect benchmarking results. + +4. **Downloading Directory configuration** + +PromBench can download data from a specific directory in object storage based on a configuration file. This configuration file specifies the directory name along with the minimum and maximum timestamps. +> **Note:** Make sure the file is applied before the ```3b_prometheus-test_deployment.yaml``` . + + - **Option 1: To Download Data from a Specific Directory** + + Create a ConfigMap with the following structure: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: + bucket-config.yml: | + path: your-directory-name + minTime: block-starting-time + maxTime: block-ending-time +``` +Replace the values of ```directory```,```minTime```, and ```maxTime``` with your desired configuration. Here ```minTime``` , ```maxTime``` are the starting and ending time of TSDB block. +- **Option 2: To Skip Data Download** + +If you do not want to download data, create an empty ConfigMap with the same name: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: +``` + +5. **Deploy the Kubernetes Objects**: ```bash make resource_apply diff --git a/prombench/docs/gke.md b/prombench/docs/gke.md index dbcb4b292..279358cd6 100644 --- a/prombench/docs/gke.md +++ b/prombench/docs/gke.md @@ -96,8 +96,71 @@ Run Prombench tests in [Google Kubernetes Engine (GKE)](https://cloud.google.com ```bash make node_create ``` - -3. **Deploy the Kubernetes Objects**: +3. **Setting Up Benchmarking Data** + When setting up a benchmarking environment, it’s often useful to have pre-generated data available.This data can help speed up testing and make benchmarks more realistic by simulating actual workloads. + +In this setup, you have two choices: + +Here’s how each option works: +- **Option 1: Download data from object storage** + + To download data from object storage, create a Kubernetes secret with exact named `bucket-secret` and file name `object-config.yml` with the necessary credentials as per your object storage. This secret enables access to the stored data. +> Note: Make sure this secret applied before `3b_prometheus-test_deployment.yaml` + +- **Option 2: Skip downloading data** + +If you don’t Want to Download data create an empty secret like this - + +```yaml +# Empty Secret to Skip Downloading Data +apiVersion: v1 +kind: Secret +metadata: + name: bucket-secret + namespace: prombench-{{ .PR_NUMBER }} +type: Opaque +stringData: + object-config.yml: +``` + +Regardless of the option chosen, data stored in Prometheus will only be retained based on the configured retention settings (```--storage.tsdb.retention.size```). + +> **⚠️ Warning:** The benchmark will change its basis when the retention size limit is reached and older downloaded blocks are deleted. Ensure that you have sufficient retention settings configured to avoid data loss that could affect benchmarking results. + +4. **Downloading Directory configuration** + +PromBench can download data from a specific directory in object storage based on a configuration file. This configuration file specifies the directory name along with the minimum and maximum timestamps. +> **Note:** Make sure the file is applied before the ```3b_prometheus-test_deployment.yaml``` . + + - **Option 1: To Download Data from a Specific Directory** + + Create a ConfigMap with the following structure: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: + bucket-config.yml: | + path: your-directory-name + minTime: block-starting-time + maxTime: block-ending-time +``` +Replace the values of ```directory```,```minTime```, and ```maxTime``` with your desired configuration. Here ```minTime``` , ```maxTime``` are the starting and ending time of TSDB block. +- **Option 2: To Skip Data Download** + +If you do not want to download data, create an empty ConfigMap with the same name: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: +``` + +5. **Deploy the Kubernetes Objects**: ```bash make resource_apply diff --git a/prombench/docs/kind.md b/prombench/docs/kind.md index 9b054c1b4..72c0a6500 100644 --- a/prombench/docs/kind.md +++ b/prombench/docs/kind.md @@ -114,7 +114,71 @@ If used with the GitHub integration: export PR_NUMBER= ``` -2. Deploy the Kubernetes objects: +2. Setting Up Benchmarking Data + When setting up a benchmarking environment, it’s often useful to have pre-generated data available.This data can help speed up testing and make benchmarks more realistic by simulating actual workloads. + +In this setup, you have two choices: + +Here’s how each option works: +- **Option 1: Download data from object storage** + + To download data from object storage, create a Kubernetes secret with exact named `bucket-secret` and file name `object-config.yml` with the necessary credentials as per your object storage. This secret enables access to the stored data. +> Note: Make sure this secret applied before `3b_prometheus-test_deployment.yaml` + +- **Option 2: Skip downloading data** + +If you don’t Want to Download data create an empty secret like this - + +```yaml +# Empty Secret to Skip Downloading Data +apiVersion: v1 +kind: Secret +metadata: + name: bucket-secret + namespace: prombench-{{ .PR_NUMBER }} +type: Opaque +stringData: + object-config.yml: +``` + +Regardless of the option chosen, data stored in Prometheus will only be retained based on the configured retention settings (```--storage.tsdb.retention.size```). + +> **⚠️ Warning:** The benchmark will change its basis when the retention size limit is reached and older downloaded blocks are deleted. Ensure that you have sufficient retention settings configured to avoid data loss that could affect benchmarking results. + +3. Downloading Directory configuration + +PromBench can download data from a specific directory in object storage based on a configuration file. This configuration file specifies the directory name along with the minimum and maximum timestamps. +> **Note:** Make sure the file is applied before the ```3b_prometheus-test_deployment.yaml``` . + + - **Option 1: To Download Data from a Specific Directory** + + Create a ConfigMap with the following structure: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: + bucket-config.yml: | + path: your-directory-name + minTime: block-starting-time + maxTime: block-ending-time +``` +Replace the values of ```directory```,```minTime```, and ```maxTime``` with your desired configuration. Here ```minTime``` , ```maxTime``` are the starting and ending time of TSDB block. +- **Option 2: To Skip Data Download** + +If you do not want to download data, create an empty ConfigMap with the same name: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: blocksync-config + namespace: prombench-{{ .PR_NUMBER }} +data: +``` + +4. Deploy the Kubernetes objects: > **_Note:_** If you encounter a `too many files open` error caused by promtail, increase the default value of `/proc/sys/fs/inotify/max_user_instances` from 128 to 512: > ```bash > sudo sysctl fs.inotify.max_user_instances=512 diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml index 63f348d28..f090939a2 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml @@ -1,14 +1,3 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: blocksync-config - namespace: prombench-{{ .PR_NUMBER }} -data: - bucket-config.yml: | - path: gendata - minTime: 1729694072541 - maxTime: 1729706400000 ---- apiVersion: apps/v1 kind: Deployment metadata: From b75309e87f3e976e09de3cebb33ca52fea5448a6 Mon Sep 17 00:00:00 2001 From: Kushal Shukla Date: Wed, 25 Dec 2024 16:17:40 +0530 Subject: [PATCH 5/5] some new changes Signed-off-by: Kushal Shukla --- .../3b_prometheus-test-pr_deployment.yaml | 6 ++---- ...3b_prometheus-test-release_deployment.yaml | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml index f090939a2..45ab51bff 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-pr_deployment.yaml @@ -34,7 +34,7 @@ spec: runAsUser: 0 initContainers: - name: prometheus-builder - image: kushalshukla/builder + image: docker.io/prominfra/prometheus-builder:master imagePullPolicy: Always env: - name: PR_NUMBER @@ -48,10 +48,8 @@ spec: volumeMounts: - name: prometheus-executable mountPath: /prometheus-builder - - name: key - mountPath: /config - name: data-downloader - image: kushalshukla/writer + image: docker.io/prominfra/block-sync:master imagePullPolicy: Always volumeMounts: - name: instance-ssd diff --git a/prombench/manifests/prombench/benchmark/3b_prometheus-test-release_deployment.yaml b/prombench/manifests/prombench/benchmark/3b_prometheus-test-release_deployment.yaml index f643ad57d..54e356e3f 100644 --- a/prombench/manifests/prombench/benchmark/3b_prometheus-test-release_deployment.yaml +++ b/prombench/manifests/prombench/benchmark/3b_prometheus-test-release_deployment.yaml @@ -32,6 +32,17 @@ spec: - prometheus securityContext: runAsUser: 0 + initContainers: + - name: data-downloader + image: docker.io/prominfra/block-sync:master + imagePullPolicy: Always + volumeMounts: + - name: instance-ssd + mountPath: /data + - name: bucket-config + mountPath: /config + - name: key + mountPath: /key containers: - name: prometheus image: quay.io/prometheus/prometheus:{{ .RELEASE }} @@ -41,7 +52,8 @@ spec: "--web.external-url=http://{{ .DOMAIN_NAME }}/{{ .PR_NUMBER }}/prometheus-release", "--storage.tsdb.path=/prometheus", "--config.file=/etc/prometheus/prometheus.yml", - "--log.level=debug" + "--log.level=debug", + "--storage.tsdb.retention.size=5GB" ] resources: requests: @@ -63,6 +75,12 @@ spec: hostPath: # /mnt is where GKE keeps it's SSD. path: /mnt/disks/ssd0 + - name: bucket-config # Define the Secret volume + secret: + secretName: bucket-secret + - name: key + configMap: + name: blocksync-config terminationGracePeriodSeconds: 300 nodeSelector: node-name: prometheus-{{ .PR_NUMBER }}