diff --git a/.github/actions/ec2-create-instances/action.yml b/.github/actions/ec2-create-instances/action.yml new file mode 100644 index 000000000..05f3b6e6e --- /dev/null +++ b/.github/actions/ec2-create-instances/action.yml @@ -0,0 +1,29 @@ +name: Create EC2 instances +description: Creates EC2 instances + +inputs: + region: + description: 'The AWS region used to host the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: aws-ec2-requirements-install + name: Install the required Ansible AWS collections. + shell: bash + run: | + ./aws_ec2.sh requirements + pipx inject ansible-core boto3 botocore + working-directory: ansible + + - id: create-ec2-instances + shell: bash + run: | + ./aws_ec2.sh create ${{ inputs.region }} + working-directory: ansible + + - id: create-load-runners + shell: bash + run: ./aws_ec2.sh start ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/ec2-delete-instances/action.yml b/.github/actions/ec2-delete-instances/action.yml new file mode 100644 index 000000000..421054d4c --- /dev/null +++ b/.github/actions/ec2-delete-instances/action.yml @@ -0,0 +1,20 @@ +name: Delete EC2 instances +description: Deletes EC2 instances + +inputs: + region: + description: 'The AWS region used to delete the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: stop-ec2-instances + shell: bash + run: ./aws_ec2.sh stop ${{ inputs.region }} + working-directory: ansible + + - id: delete-load-runners + shell: bash + run: ./aws_ec2.sh delete ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/keycloak-create-dataset/action.yml b/.github/actions/keycloak-create-dataset/action.yml index fe9bbca67..010ec7102 100644 --- a/.github/actions/keycloak-create-dataset/action.yml +++ b/.github/actions/keycloak-create-dataset/action.yml @@ -14,6 +14,18 @@ inputs: clients: description: 'Number of clients to create' default: '1' + createClientForSpecificRealm: + description: 'Create client for realm' + type: boolean + default: false + clientsPerRealm: + description: 'Number of clients per Realm.' + type: string + default: '1' + realmNameForClients: + description: 'Name of the realm' + type: string + default: 'realm-0' maxWaitEntityCreation: description: 'Maximum number of seconds to wait for creation of entities' default: '300' @@ -32,3 +44,14 @@ runs: ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset working-directory: dataset + + - id: create_clients_for_realm + shell: bash + run: | + if [[ ${{ inputs.createClientForSpecificRealm }} == true ]]; then + ./dataset-import.sh -a create-clients -c ${{ inputs.clientsPerRealm }} -n ${{ inputs.realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + else + echo "Nothing should be done here. Passed parameter is: ${{ inputs.createClientForSpecificRealm }}" + fi + working-directory: dataset diff --git a/.github/actions/keycloak-create-deployment/action.yml b/.github/actions/keycloak-create-deployment/action.yml index fd0662bd8..bb1d64388 100644 --- a/.github/actions/keycloak-create-deployment/action.yml +++ b/.github/actions/keycloak-create-deployment/action.yml @@ -32,6 +32,10 @@ inputs: default: 'postgres' databaseUrl: description: 'The external URL of the database' + deployMonitoring: + description: 'Boolean showing if monitoring should be deployed' + type: boolean + default: false runs: using: "composite" @@ -56,3 +60,13 @@ runs: KC_OTEL: ${{ inputs.otel }} KC_DATABASE: ${{ inputs.database }} KC_DATABASE_URL: ${{ inputs.databaseUrl }} + + - id: install_keycloak_monitoring + shell: bash + run: | + if [[ ${{ inputs.deployMonitoring }} == true ]]; then + task monitoring + else + echo "Nothing should be done. Passed parameter is ${{ inputs.deployMonitoring }}." + fi + working-directory: provision/openshift diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml new file mode 100644 index 000000000..571d527a2 --- /dev/null +++ b/.github/workflows/rosa-scaling-benchmark.yml @@ -0,0 +1,261 @@ +name: Rosa Cluster - Scaling Benchmark + +on: + workflow_dispatch: + inputs: + clusterName: + description: 'Name of the cluster' + type: string + default: 'gh-keycloak' + region: + description: 'Name of the region where EC2 instances should be installed' + type: string + default: 'eu-west-1' + disableStickySessions: + description: 'Disable Sticky Sessions' + type: boolean + default: true + numberOfEntitiesInRealm: + description: 'Number of entities for the scenario in DB' + type: string + default: '100000' + maxWaitEntityCreation: + description: 'Maximum number of seconds to wait for creation of entities' + type: string + default: '900' + numberOfUsersPerSecond: + description: 'Initial users per second' + type: string + default: '200' + numberOfClientsPerSecond: + description: 'Initial clients per second' + type: string + default: '1000' + skipCreateDeployment: + description: 'Skip creating Keycloak deployment' + type: boolean + default: false + skipCreateDataset: + description: 'Skip creating dataset' + type: boolean + default: false + skipDeleteProject: + description: 'Skip deleting project' + type: boolean + default: false + +concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }} + +env: + PROJECT_PREFIX: runner- # same as default + PROJECT: runner-keycloak + +jobs: + run: + name: Run Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup ROSA CLI + uses: ./.github/actions/rosa-cli-setup + with: + aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} + rosa-token: ${{ secrets.ROSA_TOKEN }} + + - name: Login to OpenShift cluster + uses: ./.github/actions/oc-keycloak-login + with: + clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }} + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Cache Maven Wrapper + uses: actions/cache@v3 + with: + path: | + .mvn/wrapper/maven-wrapper.jar + key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-maven-wrapper- + + - name: Build with Maven + run: | + ./mvnw -B clean package -DskipTests -pl benchmark + tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz + mv keycloak-benchmark-* keycloak-benchmark + + - name: Allow cluster to scale + if: ${{ !inputs.skipCreateDeployment }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling + + - name: Create Keycloak deployment + if: ${{ !inputs.skipCreateDeployment }} + uses: ./.github/actions/keycloak-create-deployment + with: + projectPrefix: ${{ env.PROJECT_PREFIX }} + disableStickySessions: ${{ !inputs.disableStickySessions }} + deployMonitoring: true + + - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users + if: ${{ !inputs.skipCreateDataset }} + uses: ./.github/actions/keycloak-create-dataset + with: + project: ${{ env.PROJECT }} + users: ${{ inputs.numberOfEntitiesInRealm }} + clients: 100 + clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }} + createClientForSpecificRealm: true + maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }} + + - name: Create AWS EC2 instances + id: create_aws_ec2_instances + uses: ./.github/actions/ec2-create-instances + with: + region: ${{ inputs.region }} + + - name: Get URLs + uses: ./.github/actions/get-keycloak-url + with: + project: ${{ env.PROJECT }} + + - name: Testing memory for creating sessions + id: kcb-authorization-code-1 + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.AuthorizationCode \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ + --ramp-up=20 \ + --logout-percentage=0 \ + --measurement=600 \ + --users-per-realm=100 \ + --log-http-on-failure + echo "Benchmark is finished." + ls -al ../keycloak-benchmark/ + echo "And other Directory:" + ls -al ../keycloak-benchmark/results + working-directory: ansible + +# - name: Testing CPU usage for user logins +# id: kcb-authorization-code-2 +# run: | +# ./benchmark.sh ${{ inputs.region }} \ +# --scenario=keycloak.scenario.authentication.AuthorizationCode \ +# --server-url=${{ env.KEYCLOAK_URL }} \ +# --realm-name=realm-0 \ +# --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ +# --ramp-up=20 \ +# --logout-percentage=100 \ +# --measurement=600 \ +# --users-per-realm=100 \ +# --log-http-on-failure +# working-directory: ansible + +# - name: Testing CPU usage for client credential grants +# id: kcb-client-secret +# run: | +# ./benchmark.sh ${{ inputs.region }} \ +# --scenario=keycloak.scenario.authentication.ClientSecret \ +# --server-url=${{ env.KEYCLOAK_URL }} \ +# --realm-name=realm-0 \ +# --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \ +# --ramp-up=20 \ +# --measurement=600 \ +# --users-per-realm=100000 \ +# --log-http-on-failure +# working-directory: ansible + + - name: Archive Gatling reports + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: gatling-results + path: keycloak-benchmark/results + retention-days: 5 + + - name: Archive Summary - Testing memory for creating sessions + uses: actions/upload-artifact@v3 + with: + name: summary + path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }} + retention-days: 5 + +# - name: Archive Summary - Testing CPU usage for user logins +# uses: actions/upload-artifact@v3 +# with: +# name: summary +# path: ${{ steps.kcb-authorization-code-2.outputs.kcb_result }} +# retention-days: 5 + +# - name: Archive Summary - Testing CPU usage for client credential grants +# uses: actions/upload-artifact@v3 +# with: +# name: summary +# path: ${{ steps.kcb-client-secret.outputs.kcb_result }} +# retention-days: 5 + + - name: Stop and Delete EC2 instances + if: ${{ failure() && steps.create_aws_ec2_instances.conclusion != 'skipped' }} + uses: ./.github/actions/ec2-delete-instances + with: + region: ${{ inputs.region }} + + - name: Delete Keycloak deployment + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + uses: ./.github/actions/keycloak-delete-deployment + with: + project: ${{ env.PROJECT }} + + - name: Scale down the cluster + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling + + archive: + name: Commit results to Git repository + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - run + steps: + + - name: Checkout repository for results + uses: actions/checkout@v3 + with: + ref: 'result-data' + + - uses: actions/download-artifact@v3 + with: + name: summary + + - name: Commit result-summary + shell: bash + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + DATE_FOLDER=scalability/$(date +"%Y/%m/%d") + mkdir -p ${DATE_FOLDER} + mv *.json ${DATE_FOLDER} + git add . + git commit -m "generated" + git push + + - name: Trigger data aggregation + if: github.repository == 'keycloak/keycloak-benchmark' + env: + GH_TOKEN: ${{ github.token }} + # manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub + run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..e5451f77f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,156 @@ +name: Test EC2 connection + +on: + workflow_dispatch: + inputs: + clusterName: + description: 'Name of the cluster' + type: string + default: 'gh-keycloak' + region: + description: 'Name of the region where EC2 instances should be installed' + type: string + default: 'eu-west-1' + +concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }} + +env: + PROJECT_PREFIX: runner- # same as default + PROJECT: runner-keycloak + +jobs: + run: + name: Run Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup ROSA CLI + uses: ./.github/actions/rosa-cli-setup + with: + aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} + rosa-token: ${{ secrets.ROSA_TOKEN }} + + - name: Login to OpenShift cluster + uses: ./.github/actions/oc-keycloak-login + with: + clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }} + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Cache Maven Wrapper + uses: actions/cache@v3 + with: + path: | + .mvn/wrapper/maven-wrapper.jar + key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-maven-wrapper- + + - name: Build with Maven + run: | + ./mvnw -B clean package -DskipTests -pl benchmark + tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz + mv keycloak-benchmark-* keycloak-benchmark + + - name: Create AWS EC2 instances + uses: ./.github/actions/ec2-create-instances + with: + region: ${{ inputs.region }} + + - name: Archive EC2 inventory files + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: inventory-files + path: ansible + retention-days: 1 + + - name: Get URLs + uses: ./.github/actions/get-keycloak-url + with: + project: ${{ env.PROJECT }} + + - name: Testing memory for creating sessions + id: kcb-authorization-code-1 + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.AuthorizationCode \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=200 \ + --ramp-up=20 \ + --logout-percentage=0 \ + --measurement=600 \ + --users-per-realm=100 \ + --log-http-on-failure + working-directory: ansible + + - name: Archive Gatling reports + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: gatling-results + path: keycloak-benchmark/results + retention-days: 5 + + - name: Archive Summary - Testing memory for creating sessions + uses: actions/upload-artifact@v3 + with: + name: summary + path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }} + retention-days: 5 + +# - name: Stop and Delete EC2 instances +# if: ${{ always() }} +# uses: ./.github/actions/ec2-delete-instances +# with: +# region: ${{ inputs.region }} + + archive: + name: Commit results to Git repository + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - run + steps: + + - name: Checkout repository for results + uses: actions/checkout@v3 + with: + ref: 'result-data' + + - uses: actions/download-artifact@v3 + with: + name: summary + + - name: Commit result-summary + shell: bash + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + DATE_FOLDER=scalability/$(date +"%Y/%m/%d") + mkdir -p ${DATE_FOLDER} + mv *.json ${DATE_FOLDER} + git add . + git commit -m "generated" + git push + + - name: Trigger data aggregation + if: github.repository == 'keycloak/keycloak-benchmark' + env: + GH_TOKEN: ${{ github.token }} + # manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub + run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data diff --git a/ansible/env.yml b/ansible/env.yml new file mode 100644 index 000000000..a7010387a --- /dev/null +++ b/ansible/env.yml @@ -0,0 +1,5 @@ +cluster_size: 5 +instance_type: t3.small +instance_volume_size: 30 +kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip +kcb_heap_size: 1G diff --git a/ansible/roles/aws_ec2/tasks/create-resources.yml b/ansible/roles/aws_ec2/tasks/create-resources.yml index cedd203ac..b3885efea 100644 --- a/ansible/roles/aws_ec2/tasks/create-resources.yml +++ b/ansible/roles/aws_ec2/tasks/create-resources.yml @@ -13,7 +13,8 @@ - proto: tcp from_port: 22 to_port: 22 - cidr_ip: '{{ control_host_ip.stdout }}/32' +# cidr_ip: '{{ control_host_ip.stdout }}/32' + cidr_ip: '0.0.0.0/0' register: group no_log: "{{ no_log_sensitive }}" diff --git a/ansible/roles/benchmark/tasks/run.yml b/ansible/roles/benchmark/tasks/run.yml index 9e2b07d5f..1e6ca4287 100644 --- a/ansible/roles/benchmark/tasks/run.yml +++ b/ansible/roles/benchmark/tasks/run.yml @@ -96,5 +96,11 @@ delegate_to: localhost shell: | RESULTS_DIR=$(realpath "{{ local_results_dir }}") + if [[ "${GITHUB_OUTPUT}" != "" ]]; then + echo "kcb_result=$RESULTS_DIR" >> "${GITHUB_OUTPUT}" + fi cd "{{ local_kcb_home }}" java -cp "lib/*" io.gatling.app.Gatling -ro $RESULTS_DIR + +- debug: var=RESULTS_DIR +- debug: var=GITHUB_OUTPUT diff --git a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc index 24925cfb3..5739b9d1d 100644 --- a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc +++ b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc @@ -198,7 +198,7 @@ cd ../../ansible [source,bash,subs="+quotes"] ---- ./benchmark.sh eu-west-1 \ ---scenario=keycloak.scenario.authentication.AuthorizationCode \ +--scenario=keycloak.scenario.authentication.ClientSecret \ --server-url=${KEYCLOAK_URL} \ --realm-name=realm-0 \ --users-per-sec=**** \ diff --git a/provision/openshift/.env b/provision/openshift/.env new file mode 100644 index 000000000..c0ab94a4c --- /dev/null +++ b/provision/openshift/.env @@ -0,0 +1,10 @@ +# no KC_CPU_LIMITS set for this scenario +KC_CPU_REQUESTS=6 +KC_INSTANCES=3 +KC_DISABLE_STICKY_SESSION=true +KC_MEMORY_REQUESTS_MB=4000 +KC_MEMORY_LIMITS_MB=4000 +KC_HEAP_MAX_MB=2048 +KC_DB_POOL_INITIAL_SIZE=30 +KC_DB_POOL_MAX_SIZE=30 +KC_DB_POOL_MIN_SIZE=30