Automation of Rosa Scaling Benchmark (keycloak#444)

.github/actions/ec2-create-instances/action.yml

@@ -0,0 +1,26 @@
+name: Create EC2 instances
+description: Creates EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to host the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: aws-ec2-requirements-install
+      name: Install the required Ansible AWS collections.
+      shell: bash
+      run: ./aws_ec2.sh requirements
+      working-directory: ansible
+
+    - id: create-ec2-instances
+      shell: bash
+      run: ./aws_ec2.sh create ${{ inputs.region }}
+      working-directory: ansible
+
+    - id: create-load-runners
+      shell: bash
+      run: ./aws_ec2.sh start ${{ inputs.region }}
+      working-directory: ansible

.github/actions/ec2-delete-instances/action.yml

@@ -0,0 +1,20 @@
+name: Delete EC2 instances
+description: Deletes EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to delete the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: stop-ec2-instances
+      shell: bash
+      run: ./aws_ec2.sh stop ${{ inputs.region }}
+      working-directory: ansible
+
+    - id: delete-load-runners
+      shell: bash
+      run: ./aws_ec2.sh delete ${{ inputs.region }}
+      working-directory: ansible

.github/actions/keycloak-create-dataset/action.yml

@@ -14,6 +14,14 @@ inputs:
   clients:
     description: 'Number of clients to create'
     default: '1'
+  createClientForSpecificRealm:
+    description: 'Create client for realm'
+    type: boolean
+    default: false
+  realmNameForClients:
+    description: 'Name of the realm'
+    type: string
+    default: 'realm-0'
   maxWaitEntityCreation:
     description: 'Maximum number of seconds to wait for creation of entities'
     default: '300'
@@ -32,3 +40,11 @@ runs:
         ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
         ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
       working-directory: dataset
+
+    - id: create_clients_for_realm
+      if:  ${{ inputs.createClientForSpecificRealm }}
+      shell: bash
+      run: |
+        ./dataset-import.sh -a create-clients -c ${{ inputs.clients }} -n ${{ realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+        ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+      working-directory: dataset

.github/workflows/rosa-scaling-benchmark.yml

@@ -0,0 +1,255 @@
+name: Rosa - Scaling benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      clusterName:
+        description: 'Name of the cluster'
+        type: string
+        default: 'gh-keycloak'
+      region:
+        description: 'Name of the region where EC2 instances should be installed'
+        type: string
+        default: 'eu-west-1'
+      disableStickySessions:
+        description: 'Disable Sticky Sessions'
+        type: boolean
+        default: true
+      numberOfEntitiesInRealm:
+        description: 'Number of entities for the scenario in DB'
+        type: string
+        default: '100000'
+      maxWaitEntityCreation:
+        description: 'Maximum number of seconds to wait for creation of entities'
+        type: string
+        default: '900'
+      numberOfUsersPerSecond:
+        description: 'Initial users per second'
+        type: string
+        default: '100000'
+      numberOfClientsPerSecond:
+          description: 'Initial clients per second'
+          type: string
+          default: '100000'
+      skipCreateDeployment:
+        description: 'Skip creating Keycloak deployment'
+        type: boolean
+        default: false
+      skipCreateDataset:
+        description: 'Skip creating dataset'
+        type: boolean
+        default: false
+      skipDeleteProject:
+        description: 'Skip deleting project'
+        type: boolean
+        default: false
+
+concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+env:
+  PROJECT_PREFIX: runner- # same as default
+  PROJECT: runner-keycloak
+
+jobs:
+  run:
+    name: Run Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup ROSA CLI
+        uses: ./.github/actions/rosa-cli-setup
+        with:
+          aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-default-region: ${{ vars.AWS_DEFAULT_REGION }}
+          rosa-token: ${{ secrets.ROSA_TOKEN }}
+
+      - name: Login to OpenShift cluster
+        uses: ./.github/actions/oc-keycloak-login
+        with:
+          clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Cache Maven Wrapper
+        uses: actions/cache@v3
+        with:
+          path: |
+            .mvn/wrapper/maven-wrapper.jar
+          key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-wrapper-
+
+      - name: Build with Maven
+        run: |
+          ./mvnw -B clean package -DskipTests -pl benchmark
+          tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz
+          mv keycloak-benchmark-* keycloak-benchmark
+
+      - name: Allow cluster to scale
+        if: ${{ !inputs.skipCreateDeployment }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling
+
+      - name: Create Keycloak deployment
+        if: ${{ !inputs.skipCreateDeployment }}
+        uses: ./.github/actions/keycloak-create-deployment
+        with:
+          projectPrefix: ${{ env.PROJECT_PREFIX }}
+          disableStickySessions: ${{ !inputs.disableStickySessions }}
+
+      - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users
+        if: "!inputs.skipCreateDataset"
+        uses: ./.github/actions/keycloak-create-dataset
+        with:
+          project: ${{ env.PROJECT }}
+          users: ${{ inputs.numberOfEntitiesInRealm }}
+          createClientForSpecificRealm: true
+          maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }}
+
+      - name: Create AWS EC2 instances
+        if: ${{ !inputs.skipCreateDeployment }}
+        uses: ./.github/actions/ec2-create-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Get URLs
+        uses: ./.github/actions/get-keycloak-url
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Testing memory for creating sessions
+        id: kcb-authorization-code-1
+        run: |
+          ./benchmark.sh ${{ inputs.region }} \
+          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=0 \
+          --measurement=600 \
+          --users-per-realm=100000 \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Testing CPU usage for user logins
+        id: kcb-authorization-code-2
+        run: |
+          ./benchmark.sh ${{ inputs.region }}  \
+          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=100 \
+          --measurement=600 \
+          --users-per-realm=100000 \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Testing CPU usage for client credential grants
+        id: kcb-client-secret
+        run: |
+          ./benchmark.sh ${{ inputs.region }} \
+          --scenario=keycloak.scenario.authentication.ClientSecret \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=100 \
+          --measurement=600 \
+          --users-per-realm=100000 \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Archive Gatling reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: gatling-results
+          path: keycloak-benchmark/results
+          retention-days: 5
+
+      - name: Archive Summary - Testing memory for creating sessions
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }}
+          retention-days: 5
+
+      - name: Archive Summary - Testing CPU usage for user logins
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }}
+          retention-days: 5
+
+      - name: Archive Summary - Testing CPU usage for client credential grants
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb-client-secret.outputs.kcb_result }}
+          retention-days: 5
+
+      - name: Stop and Delete EC2 instances
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        uses: ./.github/actions/ec2-delete-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Delete Keycloak deployment
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        uses: ./.github/actions/keycloak-delete-deployment
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Scale down the cluster
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling
+
+  archive:
+    name: Commit results to Git repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+      - run
+    steps:
+
+      - name: Checkout repository for results
+        uses: actions/checkout@v3
+        with:
+          ref: 'result-data'
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: summary
+
+      - name: Commit result-summary
+        shell: bash
+        env:
+          GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          DATE_FOLDER=scalability/$(date +"%Y/%m/%d")
+          mkdir -p ${DATE_FOLDER}
+          mv *.json ${DATE_FOLDER}
+          git add .
+          git commit -m "generated"
+          git push
+
+      - name: Trigger data aggregation
+        if: github.repository == 'keycloak/keycloak-benchmark'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        # manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub
+        run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data

ansible/env.yml

@@ -0,0 +1,5 @@
+cluster_size: 5
+instance_type: t3.small
+instance_volume_size: 30
+kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip
+kcb_heap_size: 1G

doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc

@@ -198,7 +198,7 @@ cd ../../ansible
 [source,bash,subs="+quotes"]
 ----
 ./benchmark.sh eu-west-1 \
---scenario=keycloak.scenario.authentication.AuthorizationCode \
+--scenario=keycloak.scenario.authentication.ClientSecret \
 --server-url=${KEYCLOAK_URL} \
 --realm-name=realm-0 \
 --users-per-sec=**<number of clients per second>** \

provision/aws/.env

@@ -0,0 +1,10 @@
+# no KC_CPU_LIMITS set for this scenario
+KC_CPU_REQUESTS=6
+KC_INSTANCES=3
+KC_DISABLE_STICKY_SESSION=true
+KC_MEMORY_REQUESTS_MB=4000
+KC_MEMORY_LIMITS_MB=4000
+KC_HEAP_MAX_MB=2048
+KC_DB_POOL_INITIAL_SIZE=30
+KC_DB_POOL_MAX_SIZE=30
+KC_DB_POOL_MIN_SIZE=30