diff --git a/.github/actions/prepare-horreum-report/action.yml b/.github/actions/prepare-horreum-report/action.yml new file mode 100644 index 000000000..f12b44924 --- /dev/null +++ b/.github/actions/prepare-horreum-report/action.yml @@ -0,0 +1,33 @@ +name: Prepare Horreum Report +description: Prepares JSON report before further processing, i.e. creates the JSON with the env data which gets from the configmap. + +inputs: + project: + description: 'The project namespace.' + required: true + +runs: + using: composite + steps: + - id: prepare-report-file + name: Create and Prepare Report File + shell: bash + # language=bash + run: | + output_file_prefix="result-" + cur_date=$(date) + cur_date_iso=$(date -d "$cur_date" --iso-8601=seconds) + cur_date_iso_compressed=$(date -d "$cur_date" '+%Y%m%d-%H%M%S') + uuid=$(uuidgen) + OUTPUT_FILE_NAME="${output_file_prefix}${cur_date_iso_compressed}-${uuid}.json" + echo "HORREUM_OUTPUT_FILE_NAME=$OUTPUT_FILE_NAME" >> $GITHUB_ENV + jq -n --arg current_date "${cur_date_iso}" --arg id "${uuid}" \ + '{"$schema": "urn:keycloak-benchmark:0.1", "uuid": ($id), "name": "ROSA Scalability Benchmark Run Results", + "start_dttm": ($current_date)}' > ${OUTPUT_FILE_NAME} + #Reading configmap with environment data + configJson=$(oc get configmap env-config -n ${{ env.PROJECT }} -o "jsonpath={ .data['environment_data\.json']}'" | rev | cut -d\' -f2- | rev | jq) + jq '. + {"context":{}}' ${OUTPUT_FILE_NAME} > tmp.json && \ + mv tmp.json ${OUTPUT_FILE_NAME} + #Putting environment parameters into JSON + jq --argjson configJson "${configJson}" '.context = ($configJson)' ${OUTPUT_FILE_NAME} > tmp.json && \ + mv tmp.json ${OUTPUT_FILE_NAME} diff --git a/.github/actions/prometheus-metrics-calc/action.yml b/.github/actions/prometheus-metrics-calc/action.yml index 4945f48d9..14cee07fa 100644 --- a/.github/actions/prometheus-metrics-calc/action.yml +++ b/.github/actions/prometheus-metrics-calc/action.yml @@ -79,22 +79,11 @@ runs: name: Stores got information in JSON file. env: CALCULATED_METRIC_VALUE: ${{ env.CALCULATED_METRIC_VALUE }} - OUTPUT_FILE_NAME: ${{ env.OUTPUT_FILE_NAME }} + HORREUM_OUTPUT_FILE_NAME: ${{ env.HORREUM_OUTPUT_FILE_NAME }} shell: bash # language=bash run: | - #Preparing and storing all information in JSON using jq library - description="${{ inputs.calculatedMetricName }} per Pod in ${{ inputs.replicas }} Pod cluster" - if [[ -z "${OUTPUT_FILE_NAME}" ]]; then - output_file_prefix="result-" - cur_date_iso=$(date --iso-8601=seconds) - cur_date_iso_compressed=$(date '+%Y%m%d-%H%M%S') - uuid=$(uuidgen) - OUTPUT_FILE_NAME="${output_file_prefix}${cur_date_iso_compressed}-${uuid}.json" - echo "OUTPUT_FILE_NAME=$OUTPUT_FILE_NAME" >> $GITHUB_ENV - jq -n --arg current_date "${cur_date_iso}" --arg id "${uuid}" \ - '{"uuid": ($id), "name": "ROSA Scalability Benchmark Run Results", "Execution Date and Time": ($current_date)}' > ${OUTPUT_FILE_NAME} - fi - jq --arg testName "${{ inputs.performedTestName }}" --arg key "${description}" \ - --arg val ${CALCULATED_METRIC_VALUE} '. + {($testName): {($key):($val|tonumber)}}' ${OUTPUT_FILE_NAME} > tmp.json && \ - mv tmp.json ${OUTPUT_FILE_NAME} + #Storing all information in JSON using jq library + jq --arg testName "${{ inputs.performedTestName }}" --arg key "${{ inputs.calculatedMetricName }}" \ + --arg val ${CALCULATED_METRIC_VALUE} '. + {($testName): {($key):($val|tonumber)}}' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \ + mv tmp.json ${HORREUM_OUTPUT_FILE_NAME} diff --git a/.github/workflows/rosa-multi-az-cluster-create.yml b/.github/workflows/rosa-multi-az-cluster-create.yml index 4a74c4cee..14d27132b 100644 --- a/.github/workflows/rosa-multi-az-cluster-create.yml +++ b/.github/workflows/rosa-multi-az-cluster-create.yml @@ -9,6 +9,10 @@ on: region: description: 'The AWS region to create both clusters in. Defaults to "vars.AWS_DEFAULT_REGION" if omitted.' type: string + createCluster: + description: 'Property showing if the cluster should be created or only software deployment should be performed.' + type: boolean + default: true workflow_call: inputs: clusterPrefix: @@ -17,6 +21,10 @@ on: region: description: 'The AWS region to create both clusters in. Defaults to "vars.AWS_DEFAULT_REGION" if omitted.' type: string + createCluster: + description: 'Property showing if the cluster should be created or only software deployment should be performed.' + type: boolean + default: true env: CLUSTER_PREFIX: ${{ inputs.clusterPrefix || format('gh-{0}', github.repository_owner) }} @@ -33,6 +41,7 @@ jobs: - run: echo "" cluster: + if: ${{ inputs.createCluster }} needs: meta strategy: matrix: @@ -48,6 +57,7 @@ jobs: deploy-keycloak: runs-on: ubuntu-latest + if: ${{ (always() && needs.cluster.result == 'skipped' && !inputs.createCluster) || success()}} needs: [cluster] steps: @@ -112,3 +122,5 @@ jobs: KC_DB_POOL_INITIAL_SIZE: 30 KC_DB_POOL_MAX_SIZE: 30 KC_DB_POOL_MIN_SIZE: 30 + KC_DATABASE: "aurora-postgres" + MULTI_AZ: "true" diff --git a/.github/workflows/rosa-multi-az-cluster-delete.yml b/.github/workflows/rosa-multi-az-cluster-delete.yml index f7c20c043..71a8a5dba 100644 --- a/.github/workflows/rosa-multi-az-cluster-delete.yml +++ b/.github/workflows/rosa-multi-az-cluster-delete.yml @@ -35,10 +35,15 @@ jobs: with: project: runner-keycloak + - name: Set SUBDOMAIN env variable for route53 processing + run: | + echo "SUBDOMAIN=$(echo $KEYCLOAK_URL | grep -oP '(?<=client.).*?(?=.keycloak-benchmark.com)')" >> $GITHUB_ENV + - name: Delete Route53 Records run: | - SUBDOMAIN=$(echo $KEYCLOAK_URL | grep -oP '(?<=client.).*?(?=.keycloak-benchmark.com)') ./provision/aws/route53/route53_delete.sh + env: + SUBDOMAIN: ${{ env.SUBDOMAIN }} cluster1: uses: ./.github/workflows/rosa-cluster-delete.yml diff --git a/.github/workflows/rosa-multi-az-cluster-undeploy.yml b/.github/workflows/rosa-multi-az-cluster-undeploy.yml new file mode 100644 index 000000000..19c47bdc0 --- /dev/null +++ b/.github/workflows/rosa-multi-az-cluster-undeploy.yml @@ -0,0 +1,83 @@ +name: Multi-AZ Clusters - Undeploy + +on: + workflow_dispatch: + inputs: + clusterPrefix: + description: 'The prefix to be used when cleaning the clusters' + type: string + region: + description: 'The AWS region to create both clusters in. Defaults to "vars.AWS_DEFAULT_REGION" if omitted.' + type: string + skipAuroraDeletion: + description: 'Skip Aurora database deletion.' + type: boolean + default: false + +env: + CLUSTER_PREFIX: ${{ inputs.clusterPrefix || format('gh-{0}', github.repository_owner) }} + REGION: ${{ github.event.inputs.region || vars.AWS_DEFAULT_REGION }} + +jobs: + clear-keycloak-and-infinispan-deployment-with-aurora: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup ROSA CLI + uses: ./.github/actions/rosa-cli-setup + with: + aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} + rosa-token: ${{ secrets.ROSA_TOKEN }} + + - name: Setup Go Task + uses: ./.github/actions/task-setup + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + cache: maven + + - name: Login to OpenShift cluster + uses: ./.github/actions/oc-keycloak-login + with: + clusterName: ${{ inputs.clusterPrefix }}-a + + - name: Get Keycloak Domain + uses: ./.github/actions/get-keycloak-url + with: + project: runner-keycloak + + - name: Undeploy infinispan and keycloak + working-directory: provision/rosa-cross-dc + run: | + task undeploy + env: + ROSA_CLUSTER_NAME_1: ${{ env.CLUSTER_PREFIX }}-a + ROSA_CLUSTER_NAME_2: ${{ env.CLUSTER_PREFIX }}-b + + - name: Undeploy aurora database + if: ${{ !inputs.skipAuroraDeletion }} + working-directory: provision/rosa-cross-dc + run: | + task delete-aurora + env: + AURORA_CLUSTER: ${{ env.CLUSTER_PREFIX }} + AURORA_REGION: ${{ env.REGION }} + ROSA_CLUSTER_NAME_1: ${{ env.CLUSTER_PREFIX }}-a + ROSA_CLUSTER_NAME_2: ${{ env.CLUSTER_PREFIX }}-b + + - name: Set SUBDOMAIN env variable for route53 processing + run: | + echo "SUBDOMAIN=$(echo $KEYCLOAK_URL | grep -oP '(?<=client.).*?(?=.keycloak-benchmark.com)')" >> $GITHUB_ENV + + - name: Delete Route53 Records + run: | + ./provision/aws/route53/route53_delete.sh + env: + SUBDOMAIN: ${{ env.SUBDOMAIN }} diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml index 39882aaf2..bdfb360c9 100644 --- a/.github/workflows/rosa-scaling-benchmark.yml +++ b/.github/workflows/rosa-scaling-benchmark.yml @@ -201,6 +201,9 @@ jobs: with: region: ${{ inputs.region }} + - name: Prepare horreum report + uses: ./.github/actions/prepare-horreum-report + - name: Run Memory Usage Total Query Before Benchmark uses: ./.github/actions/prometheus-run-queries with: @@ -243,8 +246,8 @@ jobs: uses: ./.github/actions/prometheus-metrics-calc with: input: memory_create_sessions - performedTestName: 'Memory usage for sessions' - calculatedMetricName: 'Active sessions per 500 MB memory' #as a unit for memory calculation is chosen 500MB memory size to find out how much user sessions so much memory can handle. + performedTestName: 'memory-usage-test' + calculatedMetricName: 'active-sessions-per-500Mb-per-pod' #as a unit for memory calculation is chosen 500MB memory size to find out how much user sessions so much memory can handle. criteriaValue: ${{ steps.active_sessions_count.outputs.active_sessions }} isvCPU: false isMemory: true @@ -284,8 +287,8 @@ jobs: uses: ./.github/actions/prometheus-metrics-calc with: input: user_logins_vCpu - performedTestName: 'CPU usage for user logins' - calculatedMetricName: 'User Logins per second per 1vCPU' + performedTestName: 'cpu-usage-for-logins-test' + calculatedMetricName: 'user-logins-per-sec-per-1vCpu-per-pod' criteriaValue: ${{ inputs.numberOfUsersPerSecond }} measurementInterval: ${{ inputs.measurement }} isvCPU: true @@ -325,8 +328,8 @@ jobs: uses: ./.github/actions/prometheus-metrics-calc with: input: client_credential_grants_vCpu - performedTestName: 'CPU usage for client credential grants' - calculatedMetricName: 'Client Credential Grants per second per 1vCPU' + performedTestName: 'cpu-usage-for-credential-grants-test' + calculatedMetricName: 'credential-grants-per-sec-per-1vCpu' criteriaValue: ${{ inputs.numberOfClientsPerSecond }} measurementInterval: ${{ inputs.measurement }} isvCPU: true diff --git a/provision/common/Taskfile.yaml b/provision/common/Taskfile.yaml index fbcfbaec2..e2953f3bb 100644 --- a/provision/common/Taskfile.yaml +++ b/provision/common/Taskfile.yaml @@ -26,6 +26,8 @@ vars: KC_REMOTE_STORE_HOST: '{{default "localhost" .KC_REMOTE_STORE_HOST}}' KC_REMOTE_STORE_PORT: '{{default "11222" .KC_REMOTE_STORE_PORT}}' KC_DISABLE_STICKY_SESSION: '{{default "false" .KC_DISABLE_STICKY_SESSION}}' + MULTI_AZ: '{{default "false" .MULTI_AZ}}' + ENV_DATA_JSON_PATH: "{{.ROOT_DIR}}/../environment_data.json" tasks: @@ -73,6 +75,39 @@ tasks: - echo {{.KC_DISABLE_STICKY_SESSION}} > .task/var-KC_DISABLE_STICKY_SESSION - echo {{.KC_HOSTNAME_OVERRIDE}} > .task/var-KC_HOSTNAME_OVERRIDE - echo {{.KC_HEALTH_HOSTNAME}} > .task/var-KC_HEALTH_HOSTNAME + - | + jq -n --arg cpu_limit "{{ .KC_CPU_REQUESTS }}" --arg num_of_pods "{{ .KC_INSTANCES }}" \ + --arg sticky_sessions "{{ .KC_DISABLE_STICKY_SESSION }}" \ + --arg mem_req_per_pod "{{ .KC_MEMORY_REQUESTS_MB }}" \ + --arg mem_limit_per_pod "{{ .KC_MEMORY_LIMITS_MB }}" \ + --arg heap_init "{{ .KC_HEAP_INIT_MB }}" \ + --arg heap_max "{{ .KC_HEAP_MAX_MB }}" \ + --arg db_pool_init "{{ .KC_DB_POOL_INITIAL_SIZE }}" \ + --arg db_pool_max "{{ .KC_DB_POOL_MAX_SIZE }}" \ + --arg db_pool_min "{{ .KC_DB_POOL_MIN_SIZE }}" \ + --arg meta_spc_init "{{ .KC_METASPACE_INIT_MB }}" \ + --arg meta_spc_max "{{ .KC_METASPACE_MAX_MB }}" \ + --arg database "{{ .KC_DATABASE }}" \ + --arg is_multi_az "{{ .MULTI_AZ }}" \ + '. + { "num_of_pods": ($num_of_pods|tonumber), "cpu_limit_per_pod": ($cpu_limit|tonumber), + "sticky_session_disabled": ($sticky_sessions), + "mem_requests_per_pod": ($mem_req_per_pod|tonumber), + "mem_limit_per_pod": ($mem_limit_per_pod|tonumber), + "heap": { + "init": ($heap_init|tonumber), + "max": ($heap_max|tonumber) + }, + "db_pool": { + "init": ($db_pool_init|tonumber), + "min": ($db_pool_min|tonumber), + "max": ($db_pool_max) + }, + "metaspace": { + "init": ($meta_spc_init|tonumber), + "max": ($meta_spc_max) + }, + "is_multi_az": ($is_multi_az), + "keycloak_database": ($database) }' > {{ .ENV_DATA_JSON_PATH }} run: once sources: - .task/subtask-{{.TASK}}.yaml diff --git a/provision/infinispan/Utils.yaml b/provision/infinispan/Utils.yaml index 61ebc5579..eb6cdf800 100644 --- a/provision/infinispan/Utils.yaml +++ b/provision/infinispan/Utils.yaml @@ -255,6 +255,8 @@ tasks: - task: wait-cluster vars: NAMESPACE: "{{.OC_NAMESPACE}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_1}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_2}}" crossdc-single: internal: true @@ -305,6 +307,8 @@ tasks: - task: wait-crossdc vars: NAMESPACE: "{{.OC_NAMESPACE_2}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_1}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_2}}" crossdc: internal: true @@ -402,6 +406,8 @@ tasks: vars: ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" NAMESPACE: "{{.OC_NAMESPACE_2}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_1}}" + - echo {{.TASK}} > ".task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME_2}}" delete-infinispan: internal: true diff --git a/provision/rosa-cross-dc/Taskfile.yaml b/provision/rosa-cross-dc/Taskfile.yaml index c8aefe32d..7aab2a84f 100644 --- a/provision/rosa-cross-dc/Taskfile.yaml +++ b/provision/rosa-cross-dc/Taskfile.yaml @@ -16,6 +16,7 @@ vars: ANSIBLE_DIR: "{{.ROOT_DIR}}/../../ansible" PYTHON_DIR: "{{.ROOT_DIR}}/../../benchmark/src/main/python" BENCHMARK_DIR: "{{.ROOT_DIR}}/../../benchmark/src/main/content/bin" + ENV_DATA_JSON_PATH: "{{.ROOT_DIR}}/../environment_data.json" KEYCLOAK_MASTER_PASSWORD: sh: aws secretsmanager get-secret-value --region eu-central-1 --secret-id keycloak-master-password --query SecretString --output text --no-cli-pager @@ -62,6 +63,10 @@ tasks: ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}" - echo "Deploying Aurora Database ({{.AURORA_CLUSTER}}) in region $(cat .task/rosa-region-{{.ROSA_CLUSTER_NAME_1}})" - AURORA_REGION="{{ .AURORA_REGION }}" AURORA_INSTANCES=2 ../aws/rds/aurora_create.sh + - task: update-env-configjson-with-new-values + vars: + PROP_NAME: "database_mode" + PROP_VALUE: "single" create-global-aurora: internal: true @@ -83,6 +88,10 @@ tasks: ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" - echo "Deploying Global Aurora Database ({{.AURORA_GLOBAL_CLUSTER}}) in regions {{.AURORA_GLOBAL_REGIONS}}" - AURORA_GLOBAL_REGIONS="{{ .AURORA_GLOBAL_REGIONS }}" ../aws/rds/aurora_create_global_db.sh + - task: update-env-configjson-with-new-values + vars: + PROP_NAME: "database_mode" + PROP_VALUE: "global" delete-aurora: desc: "Deletes Aurora Database" @@ -219,6 +228,12 @@ tasks: OC_NAMESPACE_1: "{{.KC_ISPN_NAMESPACE}}" OC_NAMESPACE_2: "{{.KC_ISPN_NAMESPACE}}" CROSS_DC_HOT_ROD_PASSWORD: "{{.RS_HOT_ROD_PASSWORD | default .KEYCLOAK_MASTER_PASSWORD}}" + - task: update-env-configjson-with-ispn-data + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}" + - task: update-env-configjson-with-ispn-data + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" patch-keycloak-image: internal: true @@ -245,7 +260,6 @@ tasks: sh: cat .task/aurora-endpoint-{{.AURORA_CLUSTER}} KC_ADMIN_PASSWORD: sh: "aws secretsmanager get-secret-value --region eu-central-1 --secret-id keycloak-master-password --query SecretString --output text --no-cli-pager || echo admin" - KC_DATABASE: "aurora-postgres" KC_CUSTOM_INFINISPAN_CONFIG: "true" KC_CUSTOM_INFINISPAN_CONFIG_FILE: "config/kcb-infinispan-cache-remote-store-config.xml" KC_ISPN_CLUSTER: "infinispan" @@ -422,6 +436,12 @@ tasks: ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" KC_HOSTNAME_OVERRIDE: "{{.KC_CLIENT_URL}}" KC_HEALTH_HOSTNAME: "{{.KC_HEALTH_URL_CLUSTER_2}}" + - task: create-env-configmap + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}" + - task: create-env-configmap + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" undeploy: desc: "Undeploy Infinispan and Keycloak in a Cross-Site deployment using ROSA clusters" @@ -451,6 +471,12 @@ tasks: - task: uninstall-infinispan vars: ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}" + - task: delete-env-configmap + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_1}}" + - task: delete-env-configmap + vars: + ROSA_CLUSTER_NAME: "{{.ROSA_CLUSTER_NAME_2}}" helm-add-repos: @@ -834,3 +860,52 @@ tasks: FAILOVER_MODE="{{.FAILOVER_MODE}}" FAILOVER_DELAY="{{.FAILOVER_DELAY}}" ./kc-failover.sh + + update-env-configjson-with-ispn-data: + internal: true + requires: + vars: + - ROSA_CLUSTER_NAME + cmds: + - echo "Updating environment json file with Infinispan data" + - task: update-env-configjson-with-new-values + vars: + PROP_NAME: "ispn_deployment" + PROP_VALUE: + sh: cat "{{.ISPN_DIR}}/.task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME}}" + sources: + - "{{.ISPN_DIR}}/.task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME}}" + preconditions: + - test -f "{{.ISPN_DIR}}/.task/kubecfg/ispn-{{.ROSA_CLUSTER_NAME}}" + + update-env-configjson-with-new-values: + internal: true + requires: + vars: + - PROP_NAME + - PROP_VALUE + cmds: + - echo "Updating environment json file with {{.PROP_NAME}}" + - "jq --arg key '{{.PROP_NAME}}' --arg value '{{ .PROP_VALUE }}' '. + {($key): ($value)}' {{.ENV_DATA_JSON_PATH}} > tmp.json && mv tmp.json {{.ENV_DATA_JSON_PATH}}" + preconditions: + - test -f "{{.ENV_DATA_JSON_PATH}}" + + create-env-configmap: + internal: true + requires: + vars: + - ROSA_CLUSTER_NAME + cmds: + - echo "Creating configmap with environment data." + - KUBECONFIG="{{.ISPN_DIR}}/.task/kubecfg/{{.ROSA_CLUSTER_NAME}}" oc -n {{.KC_NAMESPACE_PREFIX}}keycloak create configmap env-config --from-file={{.ENV_DATA_JSON_PATH}} || (kubectl -n {{.KC_NAMESPACE_PREFIX}}keycloak create configmap env-config --from-file={{.ENV_DATA_JSON_PATH}}) + preconditions: + - test -f "{{.ENV_DATA_JSON_PATH}}" + + delete-env-configmap: + internal: true + requires: + vars: + - ROSA_CLUSTER_NAME + cmds: + - echo "Deleting configmap with environment data." + - bash -c "kubectl -n {{.KC_NAMESPACE_PREFIX}}keycloak delete configmap env-config || exit 0"