feat: benchmark tests email results report (#425)

* feat: send results reports via email

* feat: minor fixes

* fix: permission issue

* fix: missing env vars

* fix: line breaks in json payload

* feat: updated the documentation

* fix: tiny fix

* fix: tiny fixes

* fix: updated docs about the building images

.github/workflows/publish-image-keycloak-benchmark.yml

@@ -1,9 +1,7 @@
 name: Create and publish Benchmark Keycloak Docker image
 
 on:
-  push:
-    branches:
-      - 'feature/quarkus'
+  workflow_dispatch:
 
 env:
   GITHUB_REGISTRY: ghcr.io

benchmark/.env.example

@@ -0,0 +1,12 @@
+SERVER_URL=
+CHES_CLIENT_ID=
+CHES_CLIENT_SECRET=
+RECEPIENT=<EMAIL_ADDRESS>
+ADMIN_USERNAME=
+ADMIN_PASSWORD=
+ADDITIONAL_CONFIG="--measurement=30"
+CHES_TOKEN_URL=
+MAIL_SERVER=
+SCENARIO=keycloak.scenario.authentication.AuthorizationCode
+NAME=sso-benchmark-runner
+NAMESPACE=

benchmark/Dockerfile

@@ -17,6 +17,12 @@ RUN rm ${KEYCLOAK_BENCHMARK_VERSION}.tar.gz
 
 RUN chmod +x /app/${KEYCLOAK_BENCHMARK_VERSION}/bin/*.sh
 
-WORKDIR /app/${KEYCLOAK_BENCHMARK_VERSION}/bin
+RUN chmod -R 777 /app/${KEYCLOAK_BENCHMARK_VERSION}
 
-ENTRYPOINT ["./kcb.sh"]
+WORKDIR /app/${KEYCLOAK_BENCHMARK_VERSION}
+
+COPY ./entrypoint.sh ./
+
+RUN chmod +x /app/${KEYCLOAK_BENCHMARK_VERSION}/entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]

benchmark/Makefile

@@ -1,21 +1,26 @@
-SHELL := /usr/bin/env bash
-NAMESPACE :=
-SCENARIO := keycloak.scenario.authentication.AuthorizationCode
-SERVER_URL :=
-ADMIN_USERNAME :=
-ADMIN_PASSWORD :=
+include .env
 
-# make sure scenario is set to the correct value
-# test #1 - 34 users per second for 30 minutes (1800 seconds) with 101 users per realm and 101 clients per realm
-ADDITIONAL_CONFIG := "--users-per-sec=34 --ramp-up=300 --users-per-realm=101 --measurement=1800 --clients-per-realm=101"
+SHELL := /usr/bin/env bash
 
 # make <comand> NAMESPACE="<namespace>"
 
 .PHONY: run_job
 run_job:
-	oc -n $(NAMESPACE) process -f ./openshift/dc.yaml -p SCENARIO=$(SCENARIO) -p SERVER_URL=$(SERVER_URL) -p ADMIN_USERNAME=$(ADMIN_USERNAME) -p ADMIN_PASSWORD=$(ADMIN_PASSWORD) -p ADDITIONAL_CONFIG=$(ADDITIONAL_CONFIG)| oc -n $(NAMESPACE) apply -f -
+	oc -n $(NAMESPACE) process -f ./openshift/dc.yaml \
+	-p SCENARIO=$(SCENARIO) \
+	-p SERVER_URL=$(SERVER_URL) \
+	-p ADMIN_USERNAME=$(ADMIN_USERNAME) \
+	-p ADMIN_PASSWORD=$(ADMIN_PASSWORD) \
+	-p ADDITIONAL_CONFIG=$(ADDITIONAL_CONFIG) \
+	-p CHES_CLIENT_ID=$(CHES_CLIENT_ID) \
+	-p CHES_CLIENT_SECRET=$(CHES_CLIENT_SECRET) \
+	-p RECEPIENT=$(RECEPIENT) \
+	-p NAME=$(NAME) \
+	-p MAIL_SERVER=$(MAIL_SERVER) \
+	-p CHES_TOKEN_URL=$(CHES_TOKEN_URL) \
+	| oc -n $(NAMESPACE) apply -f -
 
 .PHONY: cleanup
 cleanup:
-	oc -n $(NAMESPACE) delete job sso-benchmark-runner
-	oc -n $(NAMESPACE) delete pvc sso-benchmark-runner-pvc
+	oc -n $(NAMESPACE) delete job $(NAME)
+	oc -n $(NAMESPACE) delete secret $(NAME)-secret

benchmark/benchmark-guide.md

@@ -0,0 +1,92 @@
+# Benchmark Guide
+
+## Building Images
+
+### Server Image
+
+- You need a keycloak server with dataset provider added to be able to use it for generating test data
+- To build such server image, run `.github/workflows/publish-image-keycloak-benchmark.yml` that builds an image using `docker/keycloak/Dockerfile-26-perf` that explicitly copies `docker/keycloak/dataset-providers/keycloak-benchmark-dataset-0.15-SNAPSHOT.jar` provider
+- Deploy keycloak server run from this image **ONLY** in a test namespace
+- After the testing is complete, uninstall the server from the namespace
+
+### Runner Image
+
+- The runner image is required if you need to run benchmark tests against test keycloak server in an openshift pod
+- The image can be built using `.github/workflows/publish-image-benchmark-runner.yml` that uses `benchmark/Dockerfile`
+- Existing image `sso-benchmark-runner:dev` can be used and if not found, re-build the image
+- The instructions for running the benchmark runner are provided [here](#running-the-tests)
+
+## Dataset
+
+- The dataset is required to pre-populate realms, clients, and users in Keycloak under test
+- The dataset comes with a jar file that embeds a provider for generating the data
+- The dataset can be invoked through API endpoints
+- The `./docker/keycloak/Dockerfile-26-perf` is used to build Keycloak image with dataset provider. To build the image run `./.github/workflows/publish-image-keycloak-benchmark.yml` if image (`sso-benchmark:dev`) doesn't exist already
+
+**DO NOT ADD THIS PROVIDER OR USE THIS IMAGE IN PROD ENVIRONMENTS**
+
+### Generate Data
+
+```sh
+export KC_BASE_URL=
+
+# create 1 realm (realm-0)
+GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-realms?count=1
+
+# create 10000 users under realm-0
+GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-users?count=10000&realm-name=realm-0
+
+# create 400 clients under realm-0
+GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-clients?count=400&realm-name=realm-0
+
+# check the status of data generation
+GET https://${KC_BASE_URL}/auth/realms/master/dataset/status
+```
+
+## Running the Tests
+
+#### Pre-requisites
+
+- Java 21 if running locally
+- Access to Openshift cluster if running in a pod
+- CHES service account
+- Test instance of keycloak pre-loaded with test data using dataset
+
+### Locally - without entrypoint.sh
+
+- Download the benchmark test suite from `https://github.com/keycloak/keycloak-benchmark/releases/download/0.15-SNAPSHOT/keycloak-benchmark-0.15-SNAPSHOT.tar.gz`
+- Extract the folder and run
+
+```sh
+export SCENARIO=
+export SERVER_URL=
+export ADMIN_USERNAME=
+export ADMIN_PASSWORD=
+
+# using 100 users and 100 clients to make 34 req/s for a duration of upto 30 mins
+./bin/kcb.sh --scenario=${SCENARIO} --server-url=${SERVER_URL}/auth --admin-username=${ADMIN_USERNAME} --admin-password=${ADMIN_PASSWORD} --users-per-sec=34 --ramp-up=300 --users-per-realm=101 --measurement=1800 --clients-per-realm=101
+```
+
+### Locally - with entrypoint.sh
+
+- Create `.env` from `.env.example` and set the appropriate values for the variables
+- Run `./entrypoint.sh`
+
+### Openshift Pod
+
+- Create `.env` from `.env.example` and set the appropriate values for the variables
+- Ensure you are logged onto the Openshift cluster
+- Run `make cleanup` to ensure old resources get deleted
+- Run `make run_job` to deploy a secret and a job that executes `entrypoint.sh` script in a pod
+
+## Reports
+
+- The html report will be generated under the `./results` directory if running locally without using `entrypoint.sh`
+- Running `entrypoint.sh` locally or in a pod would send the report via email to the email address set under `RECEPIENT` environment variable
+- Download the attachment from the email and use `base64 --decode` to decode the file
+- After the decode, you can extract the contents from the archive
+
+## References
+
+- https://www.keycloak.org/keycloak-benchmark/benchmark-guide/latest/scenario-overview
+- https://github.com/keycloak/keycloak-benchmark

benchmark/entrypoint.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Configuration
+SENDER="bcgov.sso@gov.bc.ca"
+SUBJECT="Keycloak Benchmark Results - $(date +'%Y-%m-%d %H:%M:%S')"
+BODY="Please find the attached benchmark results. You need to base64 decode the attached file before extracting it."
+RESULTS_DIR="./results"
+ATTACHMENT_NAME="results.tar.gz"
+
+./bin/kcb.sh --scenario="$SCENARIO" --server-url="$SERVER_URL" --admin-username="$ADMIN_USERNAME" --admin-password="$ADMIN_PASSWORD" $ADDITIONAL_CONFIG
+
+if [ -d "$RESULTS_DIR" ]; then
+
+  if [ -f "$ATTACHMENT_NAME" ]; then
+    rm "$ATTACHMENT_NAME"
+  fi
+
+  tar -czvf "$ATTACHMENT_NAME" "$RESULTS_DIR"
+
+  if [ $? -eq 0 ]; then
+    echo "Folder '$RESULTS_DIR' compressed successfully to '$ATTACHMENT_NAME'."
+
+    echo "Getting access token from '$CHES_TOKEN_URL'."
+
+    # Get the access token
+    ACCESS_TOKEN=$(curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "client_id=$CHES_CLIENT_ID" -d "client_secret=$CHES_CLIENT_SECRET" -d "grant_type=client_credentials" "$CHES_TOKEN_URL" | jq -r '.access_token')
+
+    BASE64_DATA=$(base64 -w 0 $ATTACHMENT_NAME)
+
+    echo '{"from": "'"$SENDER"'", "to": ["'"$RECEPIENT"'"], "subject": "'"$SUBJECT"'", "body": "'"$BODY"'", "bodyType": "text", "attachments": [{"filename": "'"$ATTACHMENT_NAME"'", "content": "'"$BASE64_DATA"'"}]}' | curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $ACCESS_TOKEN" --data-binary @- "$MAIL_SERVER"
+
+  else
+    echo "Error: Failed to compress folder '$RESULTS_DIR'."
+  fi
+else
+  echo "Folder '$RESULTS_DIR' does not exist."
+fi
+
+exit 0

benchmark/openshift/dc.yaml

@@ -2,17 +2,17 @@ kind: Template
 apiVersion: v1
 objects:
   - apiVersion: v1
-    kind: PersistentVolumeClaim
+    kind: Secret
     metadata:
-      name: ${NAME}-pvc
-    spec:
-      accessModes:
-        - ReadWriteMany
-      storageClassName: netapp-file-standard
-      volumeMode: Filesystem
-      resources:
-        requests:
-          storage: 200Mi
+      name: ${NAME}-secret
+    type: Opaque
+    stringData:
+      admin-username: ${ADMIN_USERNAME}
+      admin-password: ${ADMIN_PASSWORD}
+      ches-client-id: ${CHES_CLIENT_ID}
+      ches-client-secret: ${CHES_CLIENT_SECRET}
+      ches-token-url: ${CHES_TOKEN_URL}
+      mail-server: ${MAIL_SERVER}
   - apiVersion: batch/v1
     kind: Job
     spec:
@@ -21,30 +21,56 @@ objects:
         metadata:
           creationTimestamp: null
         spec:
-          volumes:
-            - name: ${NAME}-pvc
-              persistentVolumeClaim:
-                claimName: ${NAME}-pvc
           containers:
             - image: ${IMAGE_REPOSITORY}:${IMAGE_TAG}
               imagePullPolicy: Always
               name: ${NAME}
-              args:
-                - --scenario=${SCENARIO}
-                - --server-url=${SERVER_URL}
-                - --admin-username=${ADMIN_USERNAME}
-                - --admin-password=${ADMIN_PASSWORD}
-                - ${ADDITIONAL_CONFIG}
               resources:
                 limits:
                   cpu: 2
                   memory: 5Gi
                 requests:
                   cpu: 500m
                   memory: 500Mi
-              volumeMounts:
-                - name: ${NAME}-pvc
-                  mountPath: /app/${KEYCLOAK_BENCHMARK_VERSION}/results
+              env:
+                - name: MAIL_SERVER
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: mail-server
+                - name: CHES_TOKEN_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: ches-token-url
+                - name: CHES_CLIENT_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: ches-client-id
+                - name: CHES_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: ches-client-secret
+                - name: ADMIN_USERNAME
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: admin-username
+                - name: ADMIN_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${NAME}-secret
+                      key: admin-password
+                - name: RECEPIENT
+                  value: ${RECEPIENT}
+                - name: SCENARIO
+                  value: ${SCENARIO}
+                - name: SERVER_URL
+                  value: ${SERVER_URL}
+                - name: ADDITIONAL_CONFIG
+                  value: ${ADDITIONAL_CONFIG}
           restartPolicy: Never
     metadata:
       name: ${NAME}
@@ -53,7 +79,7 @@ objects:
         component: ${NAME}-job
 parameters:
   - name: NAME
-    value: sso-benchmark-runner
+    description: The name of the job
   - name: IMAGE_TAG
     value: dev
   - name: IMAGE_REPOSITORY
@@ -71,3 +97,13 @@ parameters:
     description: The admin password
   - name: ADDITIONAL_CONFIG
     description: Configuration set of parameters for the test
+  - name: CHES_CLIENT_ID
+    description: The CHES client ID
+  - name: CHES_CLIENT_SECRET
+    description: The CHES client secret
+  - name: CHES_TOKEN_URL
+    description: The CHES token URL
+  - name: MAIL_SERVER
+    description: The mail server
+  - name: RECEPIENT
+    description: The email address to send the report to