Merge pull request #317 from bcgov/sso-team-880

feat: k6-tests

.gitignore

@@ -8,6 +8,7 @@ yarn-error.log*
 *.key
 *.cert
 k6/env.js
+venv
 
 ##### Terraform-specific ignores.
 

k6/README.md

@@ -2,32 +2,79 @@
 
 This folder contains load tests for our sso application
 
+## Setting up
+
+**Developing and Running tests in a local environment**: If you would like to use a local environment for developing and running tests, there is a [podman-compose.yaml](./local_setup/podman-compose.yaml) file in this repository you can use to run our custom redhat image with a postgres database. To use it, from the [loca setup folder](./local_setup/), run:
+
+- `podman-compose up`
+
+**Note**: _You will need to have installed podman and podman-compose. Alternatively, you can use the same commands with docker compose, just specify the file with the -f flag._
+
+This will start our custom keycloak image on localhost:8080, you can login with credentials username=admin, password=admin. To stop the image, you can ctrl+c out (alternatively, add the -d flag to run detached), and run `podman-compose down`. To clear out the volumes, with the image stopped, run `podman volume prune` (or specify the volumes if you have additional ones to keep). The image is currently set to use `ghcr.io/bcgov/sso:7.6.25-build.1`, this can be updated as later builds come up.
+
+**Tracking stats locally**: If you would like to compare resource usage of the different tests on your local machine, there is a small electron app to graph the output of podman stats. With the podman-compose running, you can run:
+
+- `npm i`
+- `npm start`
+
+from the [podman-grapher](./local_setup/podman-grapher/) directory. This will launch a browser window graphing the CPU and Memory usage of the local keycloak docker container over time. You can run tests with it open, and save the png's if you want to check the relative differences.
+
+**Note**:  _In `podman stats`, the CPU usage is per core. So the percent used can go up to 100 * (number of machine cores)_.
+
 ## Using
 
-- Copy `env.example.js` to `env.js`. Provide credentials for a service account with permissions to create realms and users
-- Run tests with `make <testname>` or `k6 run <js file>`
+This test requires a client with a service account to run. E.g if using the default `admin-cli` client of the master realm locally, make sure the following are configured for it:
+
+- In the client settings, set the **Access Type** to confidential, and then toggle on **Service accounts enabled**.
+- Make sure that the clientID and clientSecret in [env.js](./env.js) match that client's credentials.
+
+If testing a live application, pick an appropriate client to use with a confidential service account.
+
+- Copy `env.example.js` to `env.js`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password.
+- Run tests with `k6 run <js file>`
 
 ## Tests
 
-The tests are configured to abort if more than 1% of http requests are failing, and will
-print out the maximum sessions (vus) it could handle.
+### [Active Sessions](./activeSessions.js)
+
+This test is setup to see how requesting access tokens affects the system. It can be configured with the following variables at the top of the file:
+
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests for an access token at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop requests an access token from every realm, waiting a small delay between access token requests set by the **LOOP_DELAY** variable.
+**TOTAL_REALMS** = The number of realms to create. Each loop will request an access token from all realms on an iteration. So the total number of requested access tokens by a test will be `TOTAL_REALMS * ITERATIONS_PER_LOOP * CONCURRENT_LOOPS`. Increase this number to test if requesting access tokens from different realms with different users affects performance.
+**MAX_ALLOWED_FAILURE_RATE**: The percentage of requests to allow to fail before counting the test as failed. Enter as a string of a decimal number, e.g `'0.01'` is 1%.
+**OFFLINE** Set true to request offline_access tokens.
+**LOOP_DELAY**: The amount of time to wait between token requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
+
+### [Token Introspection](./tokenIntrospection.js)
+
+Run this test to see how hitting the token introspection endpoint affects the system.
+
+The test run can be configured with the following variables at the top of the file:
+
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests to the introspection endpoint at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop will hit the introspection endpoint this number of times, waiting a small delay between requests set by the **LOOP_DELAY** variable.
+**LOOP_DELAY**: The amount of time to wait between requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
+
+### [User Info](./userInfo.js)
+
+Run this test to see how hitting the user info endpoint affects the system.
+
+The test run can be configured with the following variables at the top of the file:
 
-### Multi Realm Active Sessions
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests to the user info endpoint at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop will hit the user info endpoint this number of times, waiting a small delay between requests set by the **LOOP_DELAY** variable.
+**LOOP_DELAY**: The amount of time to wait between requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
 
-This test is designed to create a given number of active sessions spread evenly across realms. To configure,
-set the following variables at the top of `multiRealmActiveSessions.js`:
+### [Constant Rate all Flows](./constantRateAllFlows.js)
 
-- **TOTAL_ACTIVE_SESSIONS**: The approximate total sessions across all realms
-- **TOTAL_REALMS**: The total realms to spread sessions across
-- **RAMP_UP_TIME_SECONDS**: The time to ramp up to the total number of sessions
-- **HOLD_TIME_SECONDS**: The time to hold those sessions before ending the test
+Run this test to simulate fetching an access token, grabbing user info, and introspecting the token all together. This test has two scenarios, `peakProfile` and `stress`. The peak profile test is used to imitate our peak traffic running against the application for a two hour period. The stress test will ramp up traffic linearly over a 1 hour period until API requests start to fail, and then abort.
 
-### Single Realm Active Sessions
+When stress testing, the application may get saturated with requests which prevents the teardown logic from succeeding, since it depends on the keycloak API being able to receive and act on requests. In this case, the test realms will not delete properly. These realms are all prefixed with "newrealm" and will need to be deleted manually.
 
-This test is designed to create a given number of active sessions in a single realm, and an additional
-number of empty realms. To configure, set the following variables at the top of `singleRealmActiveSessions.js`:
+The test run can be configured with the following variables at the top of the file:
 
-- **TOTAL_ACTIVE_SESSIONS**: The approximate total sessions across the primary realm
-- **TOTAL_REALMS**: The total realms to create (any additional realms will be left empty)
-- **RAMP_UP_TIME_SECONDS**: The time to ramp up to the total number of sessions
-- **HOLD_TIME_SECONDS**: The time to hold those sessions before ending the test
+**TOTAL_REALMS** = The number of realms to create.
+**MAX_ALLOWED_FAILURE_RATE**: The percentage of requests to allow to fail before counting the test as failed. Enter as a string of a decimal number, e.g `'0.01'` is 1%.
+**OFFLINE** Set true to request offline_access tokens.
+**BASELINE_RATE**: If running the peakProfile scenario, this is the peak rate per minute of requests to use. It will also determine the start rate of the stress test.

k6/activeSessions.js

@@ -0,0 +1,64 @@
+import { sleep } from 'k6';
+import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken } from './helpers.js';
+import { user } from './constants.js';
+import { username, password, clientId } from './env.js';
+
+// Alter configuration to run separate tests. See this test in the readme for configuration details.
+const CONCURRENT_LOOPS = 5;
+const ITERATIONS_PER_LOOP = 50;
+const TOTAL_REALMS = 3;
+const MAX_ALLOWED_FAILURE_RATE = '0.01';
+const OFFLINE = false;
+const LOOP_DELAY = 0.1
+
+export const options = {
+  scenarios: {
+    synchronousExecutions: {
+      executor: 'per-vu-iterations',
+      vus: CONCURRENT_LOOPS,
+      iterations: ITERATIONS_PER_LOOP,
+    },
+  },
+  thresholds: {
+    http_req_failed: [
+      {
+        threshold: `rate<${MAX_ALLOWED_FAILURE_RATE}`,
+        // Set true if you want to exit at this threshold. Can be useful for heavy tests where you want to save the poor server if its failing.
+        // abortOnFail: true,
+      },
+    ],
+  },
+};
+
+export function setup() {
+  const accessToken = getAccessToken({ username, password, clientId, confidential: true });
+  const emptyRealms = generateRealms(TOTAL_REALMS);
+  emptyRealms.forEach((realm, i) => {
+    createRealm(realm, accessToken);
+    // No spread operators allowed in k6, only es5 :(
+    const newUser = Object.assign({}, user, { username: `${user.username}_${i}` })
+    createUser(newUser, realm.realm, accessToken);
+  });
+  return emptyRealms;
+}
+
+export default function (realms) {
+  realms.forEach((realm, i) => {
+    sleep(LOOP_DELAY)
+    getAccessToken({
+      username: `${user.username}_${i}`,
+      password: user.credentials[0].value,
+      clientId,
+      confidential: true,
+      realm: realm.realm,
+      offline: OFFLINE
+    });
+  })
+}
+
+export function teardown(realms) {
+  const accessToken = getAccessToken({ username, password, clientId, confidential: true });
+  realms.forEach((realm, i) => {
+    deleteRealm(realm.realm, accessToken);
+  });
+}

k6/constantRateAllFlows.js

@@ -0,0 +1,90 @@
+import { sleep } from 'k6';
+import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken, hitIntrospectionRoute, hitUserInfoRoute, createClient } from './helpers.js';
+import { user, client } from './constants.js';
+import { username, password, clientId } from './env.js';
+
+// Alter configuration to run separate tests. See this test in the readme for configuration details.
+const TOTAL_REALMS = 1;
+// This essentially just means no dropped requests allowed since we dont get to 10000 on the peak profile.
+const MAX_ALLOWED_FAILURE_RATE = '0.0001';
+const OFFLINE = false;
+
+// Peak requests per minutes we've seen on the system
+const BASELINE_RATE = 34;
+
+export const options = {
+    scenarios: {
+        peakProfile: {
+          executor: 'constant-arrival-rate',
+          duration: '2h',
+          timeUnit: '1m',
+          rate: 34,
+          preAllocatedVUs: 5,
+        },
+        // stress: {
+        //     executor: 'ramping-arrival-rate', //Assure load increase if the system slows
+        //     startRate: BASELINE_RATE,
+        //     timeUnit: '1m',
+        //     preAllocatedVUs: 20000,
+        //     stages: [
+        //         { duration: '1m', target: BASELINE_RATE }, // just slowly ramp-up to a HUGE load
+        //         // just slowly ramp-up to an EPIC load.
+        //         { duration: '1h', target: 20000 },
+        //     ],
+        // }
+    },
+    thresholds: {
+        http_req_failed: [
+            {
+                threshold: `rate<${MAX_ALLOWED_FAILURE_RATE}`,
+                // Leave this in! Don't keep hammering the poor server after its failing, requests will queue
+                abortOnFail: true,
+            },
+        ],
+        // Requests tend to drop after 60 second timeout. Can use below to fail earlier
+        // http_req_duration: [
+        //     {
+        //         threshold: `p(95)<15000`,
+        //         abortOnFail: true,
+        //     },
+        // ]
+    },
+};
+
+export function setup() {
+    const accessToken = getAccessToken({ username, password, clientId, confidential: true });
+    const emptyRealms = generateRealms(TOTAL_REALMS);
+    emptyRealms.forEach((realm, i) => {
+        createRealm(realm, accessToken);
+        const newUser = Object.assign({}, user, { username: `${user.username}_${i}` })
+        createUser(newUser, realm.realm, accessToken);
+        // Create a confidential client to be able to use the introspection endpoint with this realm
+        createClient(realm.realm, accessToken)
+    });
+    return emptyRealms;
+}
+
+export default function (realms) {
+    realms.forEach((realm, i) => {
+        const accessToken = getAccessToken({
+            username: `${user.username}_${i}`,
+            password: user.credentials[0].value,
+            clientId,
+            confidential: true,
+            realm: realm.realm,
+            offline: OFFLINE
+        });
+        hitUserInfoRoute(accessToken, realm.realm)
+        hitIntrospectionRoute(accessToken, realm.realm, client.clientId, client.secret)
+    })
+}
+
+export function teardown(realms) {
+    // When stress testing, the enqueued requests can block teardown api requests from succeeding. Adding in a sleep to let the system recover a bit before trying to cleaunup.
+    sleep(45)
+    console.log('tearing down...')
+    const accessToken = getAccessToken({ username, password, clientId, confidential: true });
+    realms.forEach((realm, i) => {
+        deleteRealm(realm.realm, accessToken);
+    });
+}

k6/constants.js

@@ -21,7 +21,15 @@ const user = {
   credentials: [{ type: 'password', value: 'password', temporary: false }],
 };
 
+const client = {
+  secret: 'secret',
+  serviceAccountsEnabled: true,
+  clientId: 'test_privateClient',
+  directAccessGrantsEnabled: true,
+}
+
 module.exports = {
   user,
   realm,
+  client,
 }

k6/helpers.js

@@ -1,6 +1,7 @@
-import http from 'k6/http';
-import { baseUrl } from './env.js';
-import { realm } from './constants.js';
+import http, { head } from 'k6/http';
+import { baseUrl, clientId, clientSecret } from './env.js';
+import { realm, client } from './constants.js';
+import encoding from 'k6/encoding';
 
 const getHeaders = (accessToken) => ({
   Authorization: `Bearer ${accessToken}`,
@@ -17,13 +18,35 @@ function deleteRealm(realm, accessToken) {
   return http.del(`${baseUrl}/admin/realms/${realm}`, {}, { headers });
 }
 
-function getAccessToken(username, password, clientId, realm = 'master') {
-  const res = http.post(`${baseUrl}/realms/${realm}/protocol/openid-connect/token`, {
+function createClient(realm, accessToken) {
+  const headers = getHeaders(accessToken);
+  const result = http.post(`${baseUrl}/admin/realms/${realm}/clients`, JSON.stringify(client), { headers });
+
+  // Clien internal id is returned in the location header as the trailing piece or the URL.
+  const locationURIParts = result.headers.Location.split('/')
+  const clientInternalId = locationURIParts[locationURIParts.length - 1]
+  return clientInternalId
+}
+
+function deleteClient(realm, clientId, accessToken) {
+  const headers = getHeaders(accessToken);
+  http.del(`${baseUrl}/admin/realms/${realm}/clients/${clientId}`, {}, { headers });
+}
+
+function getAccessToken({username, password, clientId, confidential, realm = 'master', offline = false, secret = clientSecret}) {
+  const body = {
     grant_type: 'password',
     client_id: clientId,
     username,
     password,
-  });
+  }
+  if (confidential) {
+    body['client_secret'] = secret
+  }
+  if (offline) {
+    body['scope'] = 'email profile offline_access'
+  }
+  const res = http.post(`${baseUrl}/realms/${realm}/protocol/openid-connect/token`, body);
   try {
     return JSON.parse(res.body).access_token;
   } catch (e) {
@@ -63,6 +86,19 @@ function generateRealms(count) {
   return realms;
 }
 
+function hitUserInfoRoute(accessToken, realmName) {
+  const headers = getHeaders(accessToken)
+  const url = `${baseUrl}/realms/${realmName}/protocol/openid-connect/userinfo`
+  const result = http.get(url, { headers });
+}
+
+function hitIntrospectionRoute(accessToken, realmName, clientId, clientSecret) {
+  const base64Credentials = encoding.b64encode(`${clientId}:${clientSecret}`)
+  const url = `${baseUrl}/realms/${realmName}/protocol/openid-connect/token/introspect`;
+  const headers = { Authorization: `Basic ${base64Credentials}` }
+  http.post(url, {token: accessToken}, { headers });
+}
+
 module.exports = {
   createRealm,
   deleteRealm,
@@ -71,4 +107,8 @@ module.exports = {
   getAccessToken,
   clearRealmSessions,
   generateRealms,
+  hitUserInfoRoute,
+  hitIntrospectionRoute,
+  createClient,
+  deleteClient,
 };