Automation of Rosa Scaling Benchmark (keycloak#444)

.github/actions/ec2-create-instances/action.yml

@@ -0,0 +1,29 @@
+name: Create EC2 instances
+description: Creates EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to host the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: aws-ec2-requirements-install
+      name: Install the required Ansible AWS collections.
+      shell: bash
+      run: |
+        ./aws_ec2.sh requirements
+        pipx inject ansible-core boto3 botocore
+      working-directory: ansible
+
+    - id: create-ec2-instances
+      shell: bash
+      run: |
+        ./aws_ec2.sh create ${{ inputs.region }}
+      working-directory: ansible
+
+    - id: create-load-runners
+      shell: bash
+      run: ./aws_ec2.sh start ${{ inputs.region }}
+      working-directory: ansible

.github/actions/ec2-delete-instances/action.yml

@@ -0,0 +1,20 @@
+name: Delete EC2 instances
+description: Deletes EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to delete the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: stop-ec2-instances
+      shell: bash
+      run: ./aws_ec2.sh stop ${{ inputs.region }}
+      working-directory: ansible
+
+    - id: delete-load-runners
+      shell: bash
+      run: ./aws_ec2.sh delete ${{ inputs.region }}
+      working-directory: ansible

.github/actions/keycloak-create-dataset/action.yml

@@ -14,6 +14,18 @@ inputs:
   clients:
     description: 'Number of clients to create'
     default: '1'
+  createClientForSpecificRealm:
+    description: 'Create client for realm'
+    type: boolean
+    default: false
+  clientsPerRealm:
+    description: 'Number of clients per Realm.'
+    type: string
+    default: '1'
+  realmNameForClients:
+    description: 'Name of the realm'
+    type: string
+    default: 'realm-0'
   maxWaitEntityCreation:
     description: 'Maximum number of seconds to wait for creation of entities'
     default: '300'
@@ -32,3 +44,11 @@ runs:
         ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
         ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
       working-directory: dataset
+
+    - id: create_clients_for_realm
+      if:  ${{ inputs.createClientForSpecificRealm }}
+      shell: bash
+      run: |
+        ./dataset-import.sh -a create-clients -c ${{ inputs.clientsPerRealm }} -n ${{ inputs.realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+        ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+      working-directory: dataset

.github/actions/keycloak-create-deployment/action.yml

@@ -32,6 +32,10 @@ inputs:
     default: 'postgres'
   databaseUrl:
     description: 'The external URL of the database'
+  deployMonitoring:
+    description: 'Boolean showing if monitoring should be deployed'
+    type: boolean
+    default: false
 
 runs:
   using: "composite"
@@ -56,3 +60,13 @@ runs:
         KC_OTEL: ${{ inputs.otel }}
         KC_DATABASE: ${{ inputs.database }}
         KC_DATABASE_URL: ${{ inputs.databaseUrl }}
+
+    - id: install_keycloak_monitoring
+      shell: bash
+      run: |
+        if [[ ${{ inputs.deployMonitoring }} == true ]]; then
+          task monitoring
+        else
+          echo "Nothing should be done. Passed parameter is ${{ inputs.deployMonitoring }}."
+        fi
+      working-directory: provision/openshift

.github/workflows/rosa-scaling-benchmark.yml

@@ -0,0 +1,264 @@
+name: Rosa Cluster - Scaling Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      clusterName:
+        description: 'Name of the cluster'
+        type: string
+        default: 'gh-keycloak'
+      region:
+        description: 'Name of the region where EC2 instances should be installed'
+        type: string
+        default: 'eu-west-1'
+      disableStickySessions:
+        description: 'Disable Sticky Sessions'
+        type: boolean
+        default: true
+      numberOfEntitiesInRealm:
+        description: 'Number of entities for the scenario in DB'
+        type: string
+        default: '100000'
+      maxWaitEntityCreation:
+        description: 'Maximum number of seconds to wait for creation of entities'
+        type: string
+        default: '900'
+      numberOfUsersPerSecond:
+        description: 'Initial users per second'
+        type: string
+        default: '200'
+      numberOfClientsPerSecond:
+          description: 'Initial clients per second'
+          type: string
+          default: '1000'
+      skipCreateDeployment:
+        description: 'Skip creating Keycloak deployment'
+        type: boolean
+        default: false
+      skipCreateDataset:
+        description: 'Skip creating dataset'
+        type: boolean
+        default: false
+      skipDeleteProject:
+        description: 'Skip deleting project'
+        type: boolean
+        default: false
+
+concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+env:
+  PROJECT_PREFIX: runner- # same as default
+  PROJECT: runner-keycloak
+
+jobs:
+  run:
+    name: Run Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup ROSA CLI
+        uses: ./.github/actions/rosa-cli-setup
+        with:
+          aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-default-region: ${{ vars.AWS_DEFAULT_REGION }}
+          rosa-token: ${{ secrets.ROSA_TOKEN }}
+
+      - name: Login to OpenShift cluster
+        uses: ./.github/actions/oc-keycloak-login
+        with:
+          clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Cache Maven Wrapper
+        uses: actions/cache@v3
+        with:
+          path: |
+            .mvn/wrapper/maven-wrapper.jar
+          key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-wrapper-
+
+      - name: Build with Maven
+        run: |
+          ./mvnw -B clean package -DskipTests -pl benchmark
+          tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz
+          mv keycloak-benchmark-* keycloak-benchmark
+
+      - name: Allow cluster to scale
+        if: ${{ !inputs.skipCreateDeployment }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling
+
+      - name: Create Keycloak deployment
+        if: ${{ !inputs.skipCreateDeployment }}
+        uses: ./.github/actions/keycloak-create-deployment
+        with:
+          projectPrefix: ${{ env.PROJECT_PREFIX }}
+          disableStickySessions: ${{ !inputs.disableStickySessions }}
+          deployMonitoring: true
+
+      - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users
+        if: ${{ !inputs.skipCreateDataset }}
+        uses: ./.github/actions/keycloak-create-dataset
+        with:
+          project: ${{ env.PROJECT }}
+          users: ${{ inputs.numberOfEntitiesInRealm }}
+          clients: 100
+          clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }}
+          createClientForSpecificRealm: true
+          maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }}
+
+      - name: Create AWS EC2 instances
+        uses: ./.github/actions/ec2-create-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Get URLs
+        uses: ./.github/actions/get-keycloak-url
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Archive EC2 inventory files
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: inventory-files
+          path: ansible
+          retention-days: 1
+
+      - name: Testing memory for creating sessions
+        id: kcb-authorization-code-1
+        run: |
+          ./benchmark.sh ${{ inputs.region }} \
+          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=0 \
+          --measurement=600 \
+          --users-per-realm=100 \
+          --log-http-on-failure
+        working-directory: ansible
+
+#      - name: Testing CPU usage for user logins
+#        id: kcb-authorization-code-2
+#        run: |
+#          ./benchmark.sh ${{ inputs.region }}  \
+#          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+#          --server-url=${{ env.KEYCLOAK_URL }} \
+#          --realm-name=realm-0 \
+#          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+#          --ramp-up=20 \
+#          --logout-percentage=100 \
+#          --measurement=600 \
+#          --users-per-realm=100 \
+#          --log-http-on-failure
+#        working-directory: ansible
+
+#      - name: Testing CPU usage for client credential grants
+#        id: kcb-client-secret
+#        run: |
+#          ./benchmark.sh ${{ inputs.region }} \
+#          --scenario=keycloak.scenario.authentication.ClientSecret \
+#          --server-url=${{ env.KEYCLOAK_URL }} \
+#          --realm-name=realm-0 \
+#          --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \
+#          --ramp-up=20 \
+#          --measurement=600 \
+#          --users-per-realm=100000 \
+#          --log-http-on-failure
+#        working-directory: ansible
+
+      - name: Archive Gatling reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: gatling-results
+          path: keycloak-benchmark/results
+          retention-days: 5
+
+      - name: Archive Summary - Testing memory for creating sessions
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }}
+          retention-days: 5
+
+#      - name: Archive Summary - Testing CPU usage for user logins
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: summary
+#          path: ${{ steps.kcb-authorization-code-2.outputs.kcb_result }}
+#          retention-days: 5
+
+#      - name: Archive Summary - Testing CPU usage for client credential grants
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: summary
+#          path: ${{ steps.kcb-client-secret.outputs.kcb_result }}
+#          retention-days: 5
+
+      - name: Stop and Delete EC2 instances
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        uses: ./.github/actions/ec2-delete-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Delete Keycloak deployment
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        uses: ./.github/actions/keycloak-delete-deployment
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Scale down the cluster
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling
+
+  archive:
+    name: Commit results to Git repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+      - run
+    steps:
+
+      - name: Checkout repository for results
+        uses: actions/checkout@v3
+        with:
+          ref: 'result-data'
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: summary
+
+      - name: Commit result-summary
+        shell: bash
+        env:
+          GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          DATE_FOLDER=scalability/$(date +"%Y/%m/%d")
+          mkdir -p ${DATE_FOLDER}
+          mv *.json ${DATE_FOLDER}
+          git add .
+          git commit -m "generated"
+          git push
+
+      - name: Trigger data aggregation
+        if: github.repository == 'keycloak/keycloak-benchmark'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        # manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub
+        run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data

ansible/env.yml

@@ -0,0 +1,5 @@
+cluster_size: 5
+instance_type: t3.small
+instance_volume_size: 30
+kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip
+kcb_heap_size: 1G

ansible/roles/aws_ec2/tasks/create-resources.yml

@@ -9,11 +9,11 @@
     region: '{{ region }}'
     name: '{{ cluster_name }}'
     description: '{{ cluster_name }}'
-    rules:
-      - proto: tcp
-        from_port: 22
-        to_port: 22
-        cidr_ip: '{{ control_host_ip.stdout }}/32'
+#    rules:
+#      - proto: tcp
+#        from_port: 22
+#        to_port: 22
+#        cidr_ip: '{{ control_host_ip.stdout }}/32'
   register: group
   no_log: "{{ no_log_sensitive }}"