forked from keycloak/keycloak-benchmark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Automation of Rosa Scaling Benchmark (keycloak#444)
- Loading branch information
Anna Manukyan
committed
Aug 1, 2023
1 parent
1198b98
commit 8074ae7
Showing
7 changed files
with
343 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
name: Create EC2 instances | ||
description: Creates EC2 instances | ||
|
||
inputs: | ||
region: | ||
description: 'The AWS region used to host the EC2 instances.' | ||
required: true | ||
|
||
runs: | ||
using: composite | ||
steps: | ||
- id: aws-ec2-requirements-install | ||
name: Install the required Ansible AWS collections. | ||
shell: bash | ||
run: ./aws_ec2.sh requirements | ||
working-directory: ansible | ||
|
||
- id: create-ec2-instances | ||
shell: bash | ||
run: ./aws_ec2.sh create ${{ inputs.region }} | ||
working-directory: ansible | ||
|
||
- id: create-load-runners | ||
shell: bash | ||
run: ./aws_ec2.sh start ${{ inputs.region }} | ||
working-directory: ansible |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
name: Delete EC2 instances | ||
description: Deletes EC2 instances | ||
|
||
inputs: | ||
region: | ||
description: 'The AWS region used to delete the EC2 instances.' | ||
required: true | ||
|
||
runs: | ||
using: composite | ||
steps: | ||
- id: stop-ec2-instances | ||
shell: bash | ||
run: ./aws_ec2.sh stop ${{ inputs.region }} | ||
working-directory: ansible | ||
|
||
- id: delete-load-runners | ||
shell: bash | ||
run: ./aws_ec2.sh delete ${{ inputs.region }} | ||
working-directory: ansible |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,260 @@ | ||
name: Rosa Cluster - Scaling benchmark | ||
|
||
on: | ||
workflow_dispatch: | ||
inputs: | ||
clusterName: | ||
description: 'Name of the cluster' | ||
type: string | ||
default: 'gh-keycloak' | ||
region: | ||
description: 'Name of the region where EC2 instances should be installed' | ||
type: string | ||
default: 'eu-west-1' | ||
disableStickySessions: | ||
description: 'Disable Sticky Sessions' | ||
type: boolean | ||
default: true | ||
numberOfEntitiesInRealm: | ||
description: 'Number of entities for the scenario in DB' | ||
type: string | ||
default: '100000' | ||
clientsPerRealm: | ||
description: 'Number of clients per realm for the scenario in DB' | ||
type: string | ||
default: '100000' | ||
maxWaitEntityCreation: | ||
description: 'Maximum number of seconds to wait for creation of entities' | ||
type: string | ||
default: '900' | ||
numberOfUsersPerSecond: | ||
description: 'Initial users per second' | ||
type: string | ||
default: '100000' | ||
numberOfClientsPerSecond: | ||
description: 'Initial clients per second' | ||
type: string | ||
default: '100000' | ||
skipCreateDeployment: | ||
description: 'Skip creating Keycloak deployment' | ||
type: boolean | ||
default: false | ||
skipCreateDataset: | ||
description: 'Skip creating dataset' | ||
type: boolean | ||
default: false | ||
skipDeleteProject: | ||
description: 'Skip deleting project' | ||
type: boolean | ||
default: false | ||
|
||
concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }} | ||
|
||
env: | ||
PROJECT_PREFIX: runner- # same as default | ||
PROJECT: runner-keycloak | ||
|
||
jobs: | ||
run: | ||
name: Run Benchmark | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v3 | ||
|
||
- name: Setup ROSA CLI | ||
uses: ./.github/actions/rosa-cli-setup | ||
with: | ||
aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} | ||
rosa-token: ${{ secrets.ROSA_TOKEN }} | ||
|
||
- name: Login to OpenShift cluster | ||
uses: ./.github/actions/oc-keycloak-login | ||
with: | ||
clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }} | ||
|
||
- name: Set up JDK | ||
uses: actions/setup-java@v3 | ||
with: | ||
distribution: 'temurin' | ||
java-version: '17' | ||
cache: 'maven' | ||
|
||
- name: Cache Maven Wrapper | ||
uses: actions/cache@v3 | ||
with: | ||
path: | | ||
.mvn/wrapper/maven-wrapper.jar | ||
key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }} | ||
restore-keys: | | ||
${{ runner.os }}-maven-wrapper- | ||
- name: Build with Maven | ||
run: | | ||
./mvnw -B clean package -DskipTests -pl benchmark | ||
tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz | ||
mv keycloak-benchmark-* keycloak-benchmark | ||
- name: Allow cluster to scale | ||
if: ${{ !inputs.skipCreateDeployment }} | ||
run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling | ||
|
||
- name: Create Keycloak deployment | ||
if: ${{ !inputs.skipCreateDeployment }} | ||
uses: ./.github/actions/keycloak-create-deployment | ||
with: | ||
projectPrefix: ${{ env.PROJECT_PREFIX }} | ||
disableStickySessions: ${{ !inputs.disableStickySessions }} | ||
|
||
- name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users | ||
if: "!inputs.skipCreateDataset" | ||
uses: ./.github/actions/keycloak-create-dataset | ||
with: | ||
project: ${{ env.PROJECT }} | ||
users: ${{ inputs.numberOfEntitiesInRealm }} | ||
clientsPerRealm: ${{ inputs.clientsPerRealm }} | ||
createClientForSpecificRealm: true | ||
maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }} | ||
|
||
- name: Create AWS EC2 instances | ||
if: ${{ !inputs.skipCreateDeployment }} | ||
uses: ./.github/actions/ec2-create-instances | ||
with: | ||
region: ${{ inputs.region }} | ||
|
||
- name: Get URLs | ||
uses: ./.github/actions/get-keycloak-url | ||
with: | ||
project: ${{ env.PROJECT }} | ||
|
||
- name: Testing memory for creating sessions | ||
id: kcb-authorization-code-1 | ||
run: | | ||
./benchmark.sh ${{ inputs.region }} \ | ||
--scenario=keycloak.scenario.authentication.AuthorizationCode \ | ||
--server-url=${{ env.KEYCLOAK_URL }} \ | ||
--realm-name=realm-0 \ | ||
--users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ | ||
--ramp-up=20 \ | ||
--logout-percentage=0 \ | ||
--measurement=600 \ | ||
--users-per-realm=100000 \ | ||
--log-http-on-failure | ||
working-directory: ansible | ||
|
||
- name: Testing CPU usage for user logins | ||
id: kcb-authorization-code-2 | ||
run: | | ||
./benchmark.sh ${{ inputs.region }} \ | ||
--scenario=keycloak.scenario.authentication.AuthorizationCode \ | ||
--server-url=${{ env.KEYCLOAK_URL }} \ | ||
--realm-name=realm-0 \ | ||
--users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ | ||
--ramp-up=20 \ | ||
--logout-percentage=100 \ | ||
--measurement=600 \ | ||
--users-per-realm=100000 \ | ||
--log-http-on-failure | ||
working-directory: ansible | ||
|
||
- name: Testing CPU usage for client credential grants | ||
id: kcb-client-secret | ||
run: | | ||
./benchmark.sh ${{ inputs.region }} \ | ||
--scenario=keycloak.scenario.authentication.ClientSecret \ | ||
--server-url=${{ env.KEYCLOAK_URL }} \ | ||
--realm-name=realm-0 \ | ||
--users-per-sec=${{ inputs.numberOfClientsPerSecond }} \ | ||
--ramp-up=20 \ | ||
--logout-percentage=100 \ | ||
--measurement=600 \ | ||
--users-per-realm=100000 \ | ||
--log-http-on-failure | ||
working-directory: ansible | ||
|
||
- name: Archive Gatling reports | ||
if: ${{ always() }} | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: gatling-results | ||
path: keycloak-benchmark/results | ||
retention-days: 5 | ||
|
||
- name: Archive Summary - Testing memory for creating sessions | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: summary | ||
path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }} | ||
retention-days: 5 | ||
|
||
- name: Archive Summary - Testing CPU usage for user logins | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: summary | ||
path: ${{ steps.kcb-authorization-code-1.outputs.kcb_result }} | ||
retention-days: 5 | ||
|
||
- name: Archive Summary - Testing CPU usage for client credential grants | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: summary | ||
path: ${{ steps.kcb-client-secret.outputs.kcb_result }} | ||
retention-days: 5 | ||
|
||
- name: Stop and Delete EC2 instances | ||
if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} | ||
uses: ./.github/actions/ec2-delete-instances | ||
with: | ||
region: ${{ inputs.region }} | ||
|
||
- name: Delete Keycloak deployment | ||
if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} | ||
uses: ./.github/actions/keycloak-delete-deployment | ||
with: | ||
project: ${{ env.PROJECT }} | ||
|
||
- name: Scale down the cluster | ||
if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} | ||
run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling | ||
|
||
archive: | ||
name: Commit results to Git repository | ||
runs-on: ubuntu-latest | ||
permissions: | ||
contents: write | ||
needs: | ||
- run | ||
steps: | ||
|
||
- name: Checkout repository for results | ||
uses: actions/checkout@v3 | ||
with: | ||
ref: 'result-data' | ||
|
||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: summary | ||
|
||
- name: Commit result-summary | ||
shell: bash | ||
env: | ||
GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} | ||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
run: | | ||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" | ||
git config --global user.name "github-actions[bot]" | ||
DATE_FOLDER=scalability/$(date +"%Y/%m/%d") | ||
mkdir -p ${DATE_FOLDER} | ||
mv *.json ${DATE_FOLDER} | ||
git add . | ||
git commit -m "generated" | ||
git push | ||
- name: Trigger data aggregation | ||
if: github.repository == 'keycloak/keycloak-benchmark' | ||
env: | ||
GH_TOKEN: ${{ github.token }} | ||
# manually trigger the run, as a push with a standard GitHub action token doesn't trigger any workflow run on GitHub | ||
run: gh workflow run -R keycloak/keycloak-benchmark aggregate-results.yaml --ref result-data |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
cluster_size: 5 | ||
instance_type: t3.small | ||
instance_volume_size: 30 | ||
kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip | ||
kcb_heap_size: 1G | ||
no_log_sensitive: yes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# no KC_CPU_LIMITS set for this scenario | ||
KC_CPU_REQUESTS=6 | ||
KC_INSTANCES=3 | ||
KC_DISABLE_STICKY_SESSION=true | ||
KC_MEMORY_REQUESTS_MB=4000 | ||
KC_MEMORY_LIMITS_MB=4000 | ||
KC_HEAP_MAX_MB=2048 | ||
KC_DB_POOL_INITIAL_SIZE=30 | ||
KC_DB_POOL_MAX_SIZE=30 | ||
KC_DB_POOL_MIN_SIZE=30 |