diff --git a/.github/actions/ec2-create-instances/action.yml b/.github/actions/ec2-create-instances/action.yml new file mode 100644 index 000000000..05f3b6e6e --- /dev/null +++ b/.github/actions/ec2-create-instances/action.yml @@ -0,0 +1,29 @@ +name: Create EC2 instances +description: Creates EC2 instances + +inputs: + region: + description: 'The AWS region used to host the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: aws-ec2-requirements-install + name: Install the required Ansible AWS collections. + shell: bash + run: | + ./aws_ec2.sh requirements + pipx inject ansible-core boto3 botocore + working-directory: ansible + + - id: create-ec2-instances + shell: bash + run: | + ./aws_ec2.sh create ${{ inputs.region }} + working-directory: ansible + + - id: create-load-runners + shell: bash + run: ./aws_ec2.sh start ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/ec2-delete-instances/action.yml b/.github/actions/ec2-delete-instances/action.yml new file mode 100644 index 000000000..421054d4c --- /dev/null +++ b/.github/actions/ec2-delete-instances/action.yml @@ -0,0 +1,20 @@ +name: Delete EC2 instances +description: Deletes EC2 instances + +inputs: + region: + description: 'The AWS region used to delete the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: stop-ec2-instances + shell: bash + run: ./aws_ec2.sh stop ${{ inputs.region }} + working-directory: ansible + + - id: delete-load-runners + shell: bash + run: ./aws_ec2.sh delete ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/keycloak-create-dataset/action.yml b/.github/actions/keycloak-create-dataset/action.yml index fe9bbca67..f036ad999 100644 --- a/.github/actions/keycloak-create-dataset/action.yml +++ b/.github/actions/keycloak-create-dataset/action.yml @@ -14,6 +14,18 @@ inputs: clients: description: 'Number of clients to create' default: '1' + createClientForSpecificRealm: + description: 'Create client for realm' + type: boolean + default: false + clientsPerRealm: + description: 'Number of clients per Realm.' + type: string + default: '1' + realmNameForClients: + description: 'Name of the realm' + type: string + default: 'realm-0' maxWaitEntityCreation: description: 'Maximum number of seconds to wait for creation of entities' default: '300' @@ -32,3 +44,11 @@ runs: ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset working-directory: dataset + + - id: create_clients_for_realm + if: ${{ inputs.createClientForSpecificRealm }} + shell: bash + run: | + ./dataset-import.sh -a create-clients -c ${{ inputs.clientsPerRealm }} -n ${{ inputs.realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + working-directory: dataset diff --git a/.github/actions/keycloak-create-deployment/action.yml b/.github/actions/keycloak-create-deployment/action.yml index fd0662bd8..bb1d64388 100644 --- a/.github/actions/keycloak-create-deployment/action.yml +++ b/.github/actions/keycloak-create-deployment/action.yml @@ -32,6 +32,10 @@ inputs: default: 'postgres' databaseUrl: description: 'The external URL of the database' + deployMonitoring: + description: 'Boolean showing if monitoring should be deployed' + type: boolean + default: false runs: using: "composite" @@ -56,3 +60,13 @@ runs: KC_OTEL: ${{ inputs.otel }} KC_DATABASE: ${{ inputs.database }} KC_DATABASE_URL: ${{ inputs.databaseUrl }} + + - id: install_keycloak_monitoring + shell: bash + run: | + if [[ ${{ inputs.deployMonitoring }} == true ]]; then + task monitoring + else + echo "Nothing should be done. Passed parameter is ${{ inputs.deployMonitoring }}." + fi + working-directory: provision/openshift diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml new file mode 100644 index 000000000..638a6ade7 --- /dev/null +++ b/.github/workflows/rosa-scaling-benchmark.yml @@ -0,0 +1,264 @@ +name: Rosa Cluster - Scaling Benchmark + +on: + workflow_dispatch: + inputs: + clusterName: + description: 'Name of the cluster' + type: string + default: 'gh-keycloak' + region: + description: 'Name of the region where EC2 instances should be installed' + type: string + default: 'eu-west-1' + disableStickySessions: + description: 'Disable Sticky Sessions' + type: boolean + default: true + numberOfEntitiesInRealm: + description: 'Number of entities for the scenario in DB' + type: string + default: '100000' + maxWaitEntityCreation: + description: 'Maximum number of seconds to wait for creation of entities' + type: string + default: '900' + numberOfUsersPerSecond: + description: 'Initial users per second' + type: string + default: '200' + numberOfClientsPerSecond: + description: 'Initial clients per second' + type: string + default: '1000' + skipCreateDeployment: + description: 'Skip creating Keycloak deployment' + type: boolean + default: false + skipCreateDataset: + description: 'Skip creating dataset' + type: boolean + default: false + skipDeleteProject: + description: 'Skip deleting project' + type: boolean + default: false + +concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }} + +env: + PROJECT_PREFIX: runner- # same as default + PROJECT: runner-keycloak + +jobs: + run: + name: Run Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup ROSA CLI + uses: ./.github/actions/rosa-cli-setup + with: + aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} + rosa-token: ${{ secrets.ROSA_TOKEN }} + + - name: Login to OpenShift cluster + uses: ./.github/actions/oc-keycloak-login + with: + clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }} + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Cache Maven Wrapper + uses: actions/cache@v3 + with: + path: | + .mvn/wrapper/maven-wrapper.jar + key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-maven-wrapper- + + - name: Build with Maven + run: | + ./mvnw -B clean package -DskipTests -pl benchmark + tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz + mv keycloak-benchmark-* keycloak-benchmark + + - name: Allow cluster to scale + if: ${{ !inputs.skipCreateDeployment }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling + + - name: Create Keycloak deployment + if: ${{ !inputs.skipCreateDeployment }} + uses: ./.github/actions/keycloak-create-deployment + with: + projectPrefix: ${{ env.PROJECT_PREFIX }} + disableStickySessions: ${{ !inputs.disableStickySessions }} + deployMonitoring: true + + - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users + if: ${{ !inputs.skipCreateDataset }} + uses: ./.github/actions/keycloak-create-dataset + with: + project: ${{ env.PROJECT }} + users: ${{ inputs.numberOfEntitiesInRealm }} + clients: 100 + clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }} + createClientForSpecificRealm: true + maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }} + + - name: Create AWS EC2 instances + uses: ./.github/actions/ec2-create-instances + with: + region: ${{ inputs.region }} + + - name: Get URLs + uses: ./.github/actions/get-keycloak-url + with: + project: ${{ env.PROJECT }} + + - name: Archive EC2 inventory files + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: inventory-files + path: ansible + retention-days: 1 + + - name: Testing memory for creating sessions + id: kcb-authorization-code-1 + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.AuthorizationCode \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ + --ramp-up=20 \ + --logout-percentage=0 \ + --measurement=600 \ + --users-per-realm=100 \ + --log-http-on-failure + working-directory: ansible + +# - name: Testing CPU usage for user logins +# id: kcb-authorization-code-2 +# run: | +# ./benchmark.sh ${{ inputs.region }} \ +# --scenario=keycloak.scenario.authentication.AuthorizationCode \ +# --server-url=${{ env.KEYCLOAK_URL }} \ +# --realm-name=realm-0 \ +# --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ +# --ramp-up=20 \ +# --logout-percentage=100 \ +# --measurement=600 \ +# --users-per-realm=100 \ +# --log-http-on-failure +# working-directory: ansible + +# - name: Testing CPU usage for client credential grants +# id: kcb-client-secret +# run: | +# ./benchmark.sh ${{ inputs.region }} \ +# --scenario=keycloak.scenario.authentication.ClientSecret \ +# --server-url=${{ env.KEYCLOAK_URL }} \ +# --realm-name=realm-0 \ +# --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \ +# --ramp-up=20 \ +# --measurement=600 \ +# --users-per-realm=100000 \ +# --log-http-on-failure +# working-directory: ansible + + - name: Archive Gatling reports + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: gatling-results + path: keycloak-benchmark/results + retention-days: 5 + + - name: Archive Summary - Testing memory for creating sessions + uses: actions/upload-artifact@v3 + with: + name: summary + path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }} + retention-days: 5 + +# - name: Archive Summary - Testing CPU usage for user logins +# uses: actions/upload-artifact@v3 +# with: +# name: summary +# path: ${{ steps.kcb-authorization-code-2.outputs.kcb_result }} +# retention-days: 5 + +# - name: Archive Summary - Testing CPU usage for client credential grants +# uses: actions/upload-artifact@v3 +# with: +# name: summary +# path: ${{ steps.kcb-client-secret.outputs.kcb_result }} +# retention-days: 5 + + - name: Stop and Delete EC2 instances + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + uses: ./.github/actions/ec2-delete-instances + with: + region: ${{ inputs.region }} + + - name: Delete Keycloak deployment + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + uses: ./.github/actions/keycloak-delete-deployment + with: + project: ${{ env.PROJECT }} + + - name: Scale down the cluster + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling + + archive: + name: Commit results to Git repository + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - run + steps: + + - name: Checkout repository for results + uses: actions/checkout@v3 + with: + ref: 'result-data' + + - uses: actions/download-artifact@v3 + with: + name: summary + + - name: Commit result-summary + shell: bash + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + DATE_FOLDER=scalability/$(date +"%Y/%m/%d") + mkdir -p ${DATE_FOLDER} + mv *.json ${DATE_FOLDER} + git add . + git commit -m "generated" + git push + + - name: Trigger data aggregation + if: github.repository == 'keycloak/keycloak-benchmark' + env: + GH_TOKEN: ${{ github.token }} + # manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub + run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data diff --git a/ansible/env.yml b/ansible/env.yml new file mode 100644 index 000000000..a7010387a --- /dev/null +++ b/ansible/env.yml @@ -0,0 +1,5 @@ +cluster_size: 5 +instance_type: t3.small +instance_volume_size: 30 +kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip +kcb_heap_size: 1G diff --git a/ansible/roles/aws_ec2/tasks/create-resources.yml b/ansible/roles/aws_ec2/tasks/create-resources.yml index cedd203ac..61e13f60c 100644 --- a/ansible/roles/aws_ec2/tasks/create-resources.yml +++ b/ansible/roles/aws_ec2/tasks/create-resources.yml @@ -9,11 +9,11 @@ region: '{{ region }}' name: '{{ cluster_name }}' description: '{{ cluster_name }}' - rules: - - proto: tcp - from_port: 22 - to_port: 22 - cidr_ip: '{{ control_host_ip.stdout }}/32' +# rules: +# - proto: tcp +# from_port: 22 +# to_port: 22 +# cidr_ip: '{{ control_host_ip.stdout }}/32' register: group no_log: "{{ no_log_sensitive }}" diff --git a/ansible/roles/benchmark/tasks/run.yml b/ansible/roles/benchmark/tasks/run.yml index ee7dd3d68..9c1ae057d 100644 --- a/ansible/roles/benchmark/tasks/run.yml +++ b/ansible/roles/benchmark/tasks/run.yml @@ -13,7 +13,7 @@ - debug: var=local_results_dir - set_fact: node_count="{{ groups['benchmark'] | length }}" - + - set_fact: kcb_params_workload_per_node="{{ kcb_params }}" - when: '"--concurrent-users=" in kcb_params' block: @@ -29,13 +29,13 @@ - set_fact: users_per_sec_per_node="{{ (kcb_params | regex_search('--users-per-sec=(\S+)', '\\1') | join('') | int) / (node_count|int) }}" - debug: var=users_per_sec_per_node - set_fact: kcb_params_workload_per_node="{{ kcb_params | regex_replace('--users-per-sec=(\S+)', '--users-per-sec='+users_per_sec_per_node) }}" - + - debug: var=kcb_params_workload_per_node - name: Clean up the results directory on the remote hosts when: kcb_clean_results - file: + file: path: "{{ kcb_home }}/results" state: absent @@ -43,18 +43,19 @@ shell: | export CONFIG_ARGS=-Xmx{{ kcb_heap_size }} cd {{ kcb_home }}/bin - ./kcb.sh {{ kcb_params_workload_per_node }} > gatling.log + debug: var=kcb_params_workload_per_node + ./kcb.sh {{ kcb_params_workload_per_node }} ignore_errors: yes register: gatling_result - debug: var=gatling_result - + - name: Create local results dir delegate_to: localhost file: path="{{ local_results_dir }}" state=directory - + - name: Fetch the Gatling run return codes block: - - template: + - template: src: gatling.j2.rc dest: "{{ kcb_home }}/bin/gatling.rc" - name: Get Gatling return code @@ -62,9 +63,9 @@ src: "{{ kcb_home }}/bin/gatling.rc" dest: "{{ local_results_dir }}/gatling/{{ inventory_hostname }}.rc" flat: yes - + - name: Fetch the latest Gatling results - block: + block: - name: Find Gatling result directories find: file_type: directory diff --git a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc index 24925cfb3..5739b9d1d 100644 --- a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc +++ b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc @@ -198,7 +198,7 @@ cd ../../ansible [source,bash,subs="+quotes"] ---- ./benchmark.sh eu-west-1 \ ---scenario=keycloak.scenario.authentication.AuthorizationCode \ +--scenario=keycloak.scenario.authentication.ClientSecret \ --server-url=${KEYCLOAK_URL} \ --realm-name=realm-0 \ --users-per-sec=**** \ diff --git a/provision/aws/.env b/provision/aws/.env new file mode 100644 index 000000000..c0ab94a4c --- /dev/null +++ b/provision/aws/.env @@ -0,0 +1,10 @@ +# no KC_CPU_LIMITS set for this scenario +KC_CPU_REQUESTS=6 +KC_INSTANCES=3 +KC_DISABLE_STICKY_SESSION=true +KC_MEMORY_REQUESTS_MB=4000 +KC_MEMORY_LIMITS_MB=4000 +KC_HEAP_MAX_MB=2048 +KC_DB_POOL_INITIAL_SIZE=30 +KC_DB_POOL_MAX_SIZE=30 +KC_DB_POOL_MIN_SIZE=30