diff --git a/.github/workflows/publish-image-keycloak-benchmark.yml b/.github/workflows/publish-image-keycloak-benchmark.yml index a7e6010a..b4f57230 100644 --- a/.github/workflows/publish-image-keycloak-benchmark.yml +++ b/.github/workflows/publish-image-keycloak-benchmark.yml @@ -1,9 +1,7 @@ name: Create and publish Benchmark Keycloak Docker image on: - push: - branches: - - 'feature/quarkus' + workflow_dispatch: env: GITHUB_REGISTRY: ghcr.io diff --git a/benchmark/.env.example b/benchmark/.env.example new file mode 100644 index 00000000..48d2d887 --- /dev/null +++ b/benchmark/.env.example @@ -0,0 +1,12 @@ +SERVER_URL= +CHES_CLIENT_ID= +CHES_CLIENT_SECRET= +RECEPIENT= +ADMIN_USERNAME= +ADMIN_PASSWORD= +ADDITIONAL_CONFIG="--measurement=30" +CHES_TOKEN_URL= +MAIL_SERVER= +SCENARIO=keycloak.scenario.authentication.AuthorizationCode +NAME=sso-benchmark-runner +NAMESPACE= diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 89580182..5bc30166 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -17,6 +17,12 @@ RUN rm ${KEYCLOAK_BENCHMARK_VERSION}.tar.gz RUN chmod +x /app/${KEYCLOAK_BENCHMARK_VERSION}/bin/*.sh -WORKDIR /app/${KEYCLOAK_BENCHMARK_VERSION}/bin +RUN chmod -R 777 /app/${KEYCLOAK_BENCHMARK_VERSION} -ENTRYPOINT ["./kcb.sh"] +WORKDIR /app/${KEYCLOAK_BENCHMARK_VERSION} + +COPY ./entrypoint.sh ./ + +RUN chmod +x /app/${KEYCLOAK_BENCHMARK_VERSION}/entrypoint.sh + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/benchmark/Makefile b/benchmark/Makefile index 6d6bc1a0..6aef34be 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -1,21 +1,26 @@ -SHELL := /usr/bin/env bash -NAMESPACE := -SCENARIO := keycloak.scenario.authentication.AuthorizationCode -SERVER_URL := -ADMIN_USERNAME := -ADMIN_PASSWORD := +include .env -# make sure scenario is set to the correct value -# test #1 - 34 users per second for 30 minutes (1800 seconds) with 101 users per realm and 101 clients per realm -ADDITIONAL_CONFIG := "--users-per-sec=34 --ramp-up=300 --users-per-realm=101 --measurement=1800 --clients-per-realm=101" +SHELL := /usr/bin/env bash # make NAMESPACE="" .PHONY: run_job run_job: - oc -n $(NAMESPACE) process -f ./openshift/dc.yaml -p SCENARIO=$(SCENARIO) -p SERVER_URL=$(SERVER_URL) -p ADMIN_USERNAME=$(ADMIN_USERNAME) -p ADMIN_PASSWORD=$(ADMIN_PASSWORD) -p ADDITIONAL_CONFIG=$(ADDITIONAL_CONFIG)| oc -n $(NAMESPACE) apply -f - + oc -n $(NAMESPACE) process -f ./openshift/dc.yaml \ + -p SCENARIO=$(SCENARIO) \ + -p SERVER_URL=$(SERVER_URL) \ + -p ADMIN_USERNAME=$(ADMIN_USERNAME) \ + -p ADMIN_PASSWORD=$(ADMIN_PASSWORD) \ + -p ADDITIONAL_CONFIG=$(ADDITIONAL_CONFIG) \ + -p CHES_CLIENT_ID=$(CHES_CLIENT_ID) \ + -p CHES_CLIENT_SECRET=$(CHES_CLIENT_SECRET) \ + -p RECEPIENT=$(RECEPIENT) \ + -p NAME=$(NAME) \ + -p MAIL_SERVER=$(MAIL_SERVER) \ + -p CHES_TOKEN_URL=$(CHES_TOKEN_URL) \ + | oc -n $(NAMESPACE) apply -f - .PHONY: cleanup cleanup: - oc -n $(NAMESPACE) delete job sso-benchmark-runner - oc -n $(NAMESPACE) delete pvc sso-benchmark-runner-pvc + oc -n $(NAMESPACE) delete job $(NAME) + oc -n $(NAMESPACE) delete secret $(NAME)-secret diff --git a/benchmark/benchmark-guide.md b/benchmark/benchmark-guide.md new file mode 100644 index 00000000..08dcbcc3 --- /dev/null +++ b/benchmark/benchmark-guide.md @@ -0,0 +1,92 @@ +# Benchmark Guide + +## Building Images + +### Server Image + +- You need a keycloak server with dataset provider added to be able to use it for generating test data +- To build such server image, run `.github/workflows/publish-image-keycloak-benchmark.yml` that builds an image using `docker/keycloak/Dockerfile-26-perf` that explicitly copies `docker/keycloak/dataset-providers/keycloak-benchmark-dataset-0.15-SNAPSHOT.jar` provider +- Deploy keycloak server run from this image **ONLY** in a test namespace +- After the testing is complete, uninstall the server from the namespace + +### Runner Image + +- The runner image is required if you need to run benchmark tests against test keycloak server in an openshift pod +- The image can be built using `.github/workflows/publish-image-benchmark-runner.yml` that uses `benchmark/Dockerfile` +- Existing image `sso-benchmark-runner:dev` can be used and if not found, re-build the image +- The instructions for running the benchmark runner are provided [here](#running-the-tests) + +## Dataset + +- The dataset is required to pre-populate realms, clients, and users in Keycloak under test +- The dataset comes with a jar file that embeds a provider for generating the data +- The dataset can be invoked through API endpoints +- The `./docker/keycloak/Dockerfile-26-perf` is used to build Keycloak image with dataset provider. To build the image run `./.github/workflows/publish-image-keycloak-benchmark.yml` if image (`sso-benchmark:dev`) doesn't exist already + +**DO NOT ADD THIS PROVIDER OR USE THIS IMAGE IN PROD ENVIRONMENTS** + +### Generate Data + +```sh +export KC_BASE_URL= + +# create 1 realm (realm-0) +GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-realms?count=1 + +# create 10000 users under realm-0 +GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-users?count=10000&realm-name=realm-0 + +# create 400 clients under realm-0 +GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-clients?count=400&realm-name=realm-0 + +# check the status of data generation +GET https://${KC_BASE_URL}/auth/realms/master/dataset/status +``` + +## Running the Tests + +#### Pre-requisites + +- Java 21 if running locally +- Access to Openshift cluster if running in a pod +- CHES service account +- Test instance of keycloak pre-loaded with test data using dataset + +### Locally - without entrypoint.sh + +- Download the benchmark test suite from `https://github.com/keycloak/keycloak-benchmark/releases/download/0.15-SNAPSHOT/keycloak-benchmark-0.15-SNAPSHOT.tar.gz` +- Extract the folder and run + +```sh +export SCENARIO= +export SERVER_URL= +export ADMIN_USERNAME= +export ADMIN_PASSWORD= + +# using 100 users and 100 clients to make 34 req/s for a duration of upto 30 mins +./bin/kcb.sh --scenario=${SCENARIO} --server-url=${SERVER_URL}/auth --admin-username=${ADMIN_USERNAME} --admin-password=${ADMIN_PASSWORD} --users-per-sec=34 --ramp-up=300 --users-per-realm=101 --measurement=1800 --clients-per-realm=101 +``` + +### Locally - with entrypoint.sh + +- Create `.env` from `.env.example` and set the appropriate values for the variables +- Run `./entrypoint.sh` + +### Openshift Pod + +- Create `.env` from `.env.example` and set the appropriate values for the variables +- Ensure you are logged onto the Openshift cluster +- Run `make cleanup` to ensure old resources get deleted +- Run `make run_job` to deploy a secret and a job that executes `entrypoint.sh` script in a pod + +## Reports + +- The html report will be generated under the `./results` directory if running locally without using `entrypoint.sh` +- Running `entrypoint.sh` locally or in a pod would send the report via email to the email address set under `RECEPIENT` environment variable +- Download the attachment from the email and use `base64 --decode` to decode the file +- After the decode, you can extract the contents from the archive + +## References + +- https://www.keycloak.org/keycloak-benchmark/benchmark-guide/latest/scenario-overview +- https://github.com/keycloak/keycloak-benchmark diff --git a/benchmark/entrypoint.sh b/benchmark/entrypoint.sh new file mode 100755 index 00000000..67d151ad --- /dev/null +++ b/benchmark/entrypoint.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Configuration +SENDER="bcgov.sso@gov.bc.ca" +SUBJECT="Keycloak Benchmark Results - $(date +'%Y-%m-%d %H:%M:%S')" +BODY="Please find the attached benchmark results. You need to base64 decode the attached file before extracting it." +RESULTS_DIR="./results" +ATTACHMENT_NAME="results.tar.gz" + +./bin/kcb.sh --scenario="$SCENARIO" --server-url="$SERVER_URL" --admin-username="$ADMIN_USERNAME" --admin-password="$ADMIN_PASSWORD" $ADDITIONAL_CONFIG + +if [ -d "$RESULTS_DIR" ]; then + + if [ -f "$ATTACHMENT_NAME" ]; then + rm "$ATTACHMENT_NAME" + fi + + tar -czvf "$ATTACHMENT_NAME" "$RESULTS_DIR" + + if [ $? -eq 0 ]; then + echo "Folder '$RESULTS_DIR' compressed successfully to '$ATTACHMENT_NAME'." + + echo "Getting access token from '$CHES_TOKEN_URL'." + + # Get the access token + ACCESS_TOKEN=$(curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "client_id=$CHES_CLIENT_ID" -d "client_secret=$CHES_CLIENT_SECRET" -d "grant_type=client_credentials" "$CHES_TOKEN_URL" | jq -r '.access_token') + + BASE64_DATA=$(base64 -w 0 $ATTACHMENT_NAME) + + echo '{"from": "'"$SENDER"'", "to": ["'"$RECEPIENT"'"], "subject": "'"$SUBJECT"'", "body": "'"$BODY"'", "bodyType": "text", "attachments": [{"filename": "'"$ATTACHMENT_NAME"'", "content": "'"$BASE64_DATA"'"}]}' | curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $ACCESS_TOKEN" --data-binary @- "$MAIL_SERVER" + + else + echo "Error: Failed to compress folder '$RESULTS_DIR'." + fi +else + echo "Folder '$RESULTS_DIR' does not exist." +fi + +exit 0 diff --git a/benchmark/openshift/dc.yaml b/benchmark/openshift/dc.yaml index 0428e474..5a57de07 100644 --- a/benchmark/openshift/dc.yaml +++ b/benchmark/openshift/dc.yaml @@ -2,17 +2,17 @@ kind: Template apiVersion: v1 objects: - apiVersion: v1 - kind: PersistentVolumeClaim + kind: Secret metadata: - name: ${NAME}-pvc - spec: - accessModes: - - ReadWriteMany - storageClassName: netapp-file-standard - volumeMode: Filesystem - resources: - requests: - storage: 200Mi + name: ${NAME}-secret + type: Opaque + stringData: + admin-username: ${ADMIN_USERNAME} + admin-password: ${ADMIN_PASSWORD} + ches-client-id: ${CHES_CLIENT_ID} + ches-client-secret: ${CHES_CLIENT_SECRET} + ches-token-url: ${CHES_TOKEN_URL} + mail-server: ${MAIL_SERVER} - apiVersion: batch/v1 kind: Job spec: @@ -21,20 +21,10 @@ objects: metadata: creationTimestamp: null spec: - volumes: - - name: ${NAME}-pvc - persistentVolumeClaim: - claimName: ${NAME}-pvc containers: - image: ${IMAGE_REPOSITORY}:${IMAGE_TAG} imagePullPolicy: Always name: ${NAME} - args: - - --scenario=${SCENARIO} - - --server-url=${SERVER_URL} - - --admin-username=${ADMIN_USERNAME} - - --admin-password=${ADMIN_PASSWORD} - - ${ADDITIONAL_CONFIG} resources: limits: cpu: 2 @@ -42,9 +32,45 @@ objects: requests: cpu: 500m memory: 500Mi - volumeMounts: - - name: ${NAME}-pvc - mountPath: /app/${KEYCLOAK_BENCHMARK_VERSION}/results + env: + - name: MAIL_SERVER + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: mail-server + - name: CHES_TOKEN_URL + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: ches-token-url + - name: CHES_CLIENT_ID + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: ches-client-id + - name: CHES_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: ches-client-secret + - name: ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: admin-username + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: ${NAME}-secret + key: admin-password + - name: RECEPIENT + value: ${RECEPIENT} + - name: SCENARIO + value: ${SCENARIO} + - name: SERVER_URL + value: ${SERVER_URL} + - name: ADDITIONAL_CONFIG + value: ${ADDITIONAL_CONFIG} restartPolicy: Never metadata: name: ${NAME} @@ -53,7 +79,7 @@ objects: component: ${NAME}-job parameters: - name: NAME - value: sso-benchmark-runner + description: The name of the job - name: IMAGE_TAG value: dev - name: IMAGE_REPOSITORY @@ -71,3 +97,13 @@ parameters: description: The admin password - name: ADDITIONAL_CONFIG description: Configuration set of parameters for the test + - name: CHES_CLIENT_ID + description: The CHES client ID + - name: CHES_CLIENT_SECRET + description: The CHES client secret + - name: CHES_TOKEN_URL + description: The CHES token URL + - name: MAIL_SERVER + description: The mail server + - name: RECEPIENT + description: The email address to send the report to diff --git a/docs/benchmark-guide.md b/docs/benchmark-guide.md deleted file mode 100644 index 615c70f1..00000000 --- a/docs/benchmark-guide.md +++ /dev/null @@ -1,47 +0,0 @@ -# Benchmark Guide - -## Dataset - -- The dataset is required to pre-populate realms, clients, and users in Keycloak under test -- The dataset comes with a jar file that embeds a provider for generating the data -- The dataset can be invoked through API endpoints -- The `./docker/keycloak/Dockerfile-26-perf` is used to build Keycloak image with dataset provider. To build the image run `./.github/workflows/publish-image-keycloak-benchmark.yml` if image (`sso-benchmark:dev`) doesn't exist already - -**DO NOT ADD THIS PROVIDER OR USE THIS IMAGE IN PROD ENVIRONMENTS** - -### Generate Data - -```sh -export KC_BASE_URL= - -# create 1 realm (realm-0) -GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-realms?count=1 - -# create 10000 users under realm-0 -GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-users?count=10000&realm-name=realm-0 - -# create 400 clients under realm-0 -GET https://${KC_BASE_URL}/auth/realms/master/dataset/create-clients?count=400&realm-name=realm-0 - -# check the status of data generation -GET https://${KC_BASE_URL}/auth/realms/master/dataset/status -``` - -## Benchmark - -- Download the benchmark test suite from `https://github.com/keycloak/keycloak-benchmark/releases/download/0.15-SNAPSHOT/keycloak-benchmark-0.15-SNAPSHOT.tar.gz` -- Extract the folder and run - -```sh -# using 100 users and 100 clients to make 34 req/s for a duration of upto 30 mins -./bin/kcb.sh --scenario=keycloak.scenario.authentication.AuthorizationCode --server-url=${KC_BASE_URL}/auth --admin-username=xxx --admin-password=xxxx --users-per-sec=34 --ramp-up=300 --users-per-realm=101 --measurement=1800 --clients-per-realm=101 -``` - -## Reports - -- The html report will be generated under the `./results` directory - -## References - -- https://www.keycloak.org/keycloak-benchmark/benchmark-guide/latest/scenario-overview -- https://github.com/keycloak/keycloak-benchmark