From f54612d09906e27470ee4c6ad105be707b7da91c Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Wed, 11 Sep 2024 06:53:51 +1000 Subject: [PATCH] Refactor k6 development environment to prepare for regular runs in CD (#4903) * Add k6 to ov * Improve Openverse k6 devex with TS and ESLint * Fix args passing and allow overriding scenario options * Fix broken eslint import config --- docker/dev_env/Dockerfile | 3 + frontend/package.json | 2 +- justfile | 6 +- .../js/eslint-plugin/src/configs/import.ts | 4 +- packages/js/k6/README.md | 46 +++++ packages/js/k6/justfile | 4 + packages/js/k6/package.json | 20 +++ packages/js/k6/rollup.config.ts | 33 ++++ packages/js/k6/src/frontend/all.test.ts | 5 + packages/js/k6/src/frontend/constants.ts | 2 + packages/js/k6/src/frontend/scenarios.ts | 160 ++++++++++++++++++ packages/js/k6/src/frontend/search-en.test.ts | 5 + .../js/k6/src/frontend/search-locales.test.ts | 5 + packages/js/k6/src/frontend/static-en.test.ts | 5 + .../js/k6/src/frontend/static-locales.test.ts | 5 + packages/js/k6/src/utils.ts | 30 ++++ packages/js/k6/tsconfig.json | 35 ++++ pnpm-lock.yaml | 39 +++++ utilities/load_testing/.env.sh.template | 3 - utilities/load_testing/.gitignore | 2 - utilities/load_testing/Dockerfile | 33 ---- utilities/load_testing/README.md | 61 ------- utilities/load_testing/api.sh | 32 ---- utilities/load_testing/get_word.sh | 3 - utilities/load_testing/justfile | 38 ----- utilities/load_testing/k6/frontend.js | 139 --------------- utilities/load_testing/k6/main.js | 70 -------- utilities/load_testing/k6/observation.js | 88 ---------- utilities/load_testing/k6/search.js | 53 ------ utilities/load_testing/k6/utils.js | 48 ------ 30 files changed, 405 insertions(+), 574 deletions(-) create mode 100644 packages/js/k6/README.md create mode 100644 packages/js/k6/justfile create mode 100644 packages/js/k6/package.json create mode 100644 packages/js/k6/rollup.config.ts create mode 100644 packages/js/k6/src/frontend/all.test.ts create mode 100644 packages/js/k6/src/frontend/constants.ts create mode 100644 packages/js/k6/src/frontend/scenarios.ts create mode 100644 packages/js/k6/src/frontend/search-en.test.ts create mode 100644 packages/js/k6/src/frontend/search-locales.test.ts create mode 100644 packages/js/k6/src/frontend/static-en.test.ts create mode 100644 packages/js/k6/src/frontend/static-locales.test.ts create mode 100644 packages/js/k6/src/utils.ts create mode 100644 packages/js/k6/tsconfig.json delete mode 100644 utilities/load_testing/.env.sh.template delete mode 100644 utilities/load_testing/.gitignore delete mode 100644 utilities/load_testing/Dockerfile delete mode 100644 utilities/load_testing/README.md delete mode 100755 utilities/load_testing/api.sh delete mode 100755 utilities/load_testing/get_word.sh delete mode 100644 utilities/load_testing/justfile delete mode 100644 utilities/load_testing/k6/frontend.js delete mode 100644 utilities/load_testing/k6/main.js delete mode 100644 utilities/load_testing/k6/observation.js delete mode 100644 utilities/load_testing/k6/search.js delete mode 100644 utilities/load_testing/k6/utils.js diff --git a/docker/dev_env/Dockerfile b/docker/dev_env/Dockerfile index a44d2c83e80..d59e6ce3baf 100644 --- a/docker/dev_env/Dockerfile +++ b/docker/dev_env/Dockerfile @@ -37,6 +37,7 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}" # - nodejs: language runtime (includes npm but not Corepack) # - docker*: used to interact with host Docker socket # - postgresql*: required to connect to PostgreSQL databases +# - k6, words: used for load testing # # pipx dependencies: # - httpie: CLI HTTP client @@ -50,6 +51,7 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}" RUN mkdir /pipx \ && dnf -y install dnf-plugins-core \ && dnf -y config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ + && dnf -y install https://dl.k6.io/rpm/repo.rpm \ && dnf -y install \ git \ g++ \ @@ -63,6 +65,7 @@ RUN mkdir /pipx \ docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ unzip \ postgresql postgresql-devel \ + k6 words \ && dnf clean all \ && pipx install --global \ httpie \ diff --git a/frontend/package.json b/frontend/package.json index 8fc28995a2e..84ac65c1e33 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -41,7 +41,7 @@ "test:storybook:local": "playwright test -c test/storybook", "test:storybook:debug": "PWDEBUG=1 pnpm test:storybook:local", "test:storybook:gen": "playwright codegen localhost:54000/", - "types": "vue-tsc -p .", + "types": "pnpm run prepare:nuxt && vue-tsc -p .", "i18n": "pnpm i18n:create-locales-list && pnpm i18n:get-translations && pnpm i18n:update-locales", "i18n:en": "pnpm i18n:get-translations --en-only", "i18n:copy-test-locales": "cp test/locales/**.json src/locales/ && mv src/locales/valid-locales.json src/locales/scripts/valid-locales.json", diff --git a/justfile b/justfile index 67fb8912ceb..851b31f39ed 100644 --- a/justfile +++ b/justfile @@ -357,7 +357,7 @@ p package script +args="": # Run eslint with --fix and default file selection enabled; used to enable easy file overriding whilst retaining the defaults when running --all-files eslint *files="frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json": - just p '@openverse/eslint-plugin' run build + just p '@openverse/eslint-plugin' build pnpm exec eslint \ --ext .js,.ts,.vue,.json,.json5 \ --ignore-path .gitignore \ @@ -365,3 +365,7 @@ eslint *files="frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js pr --max-warnings=0 \ --fix \ {{ files }} + +# Alias for `just packages/js/k6/run` or `just p k6 run` +@k6 *args: + just packages/js/k6/run {{ args }} diff --git a/packages/js/eslint-plugin/src/configs/import.ts b/packages/js/eslint-plugin/src/configs/import.ts index 980aa226cd1..9cf5abb8e02 100644 --- a/packages/js/eslint-plugin/src/configs/import.ts +++ b/packages/js/eslint-plugin/src/configs/import.ts @@ -35,8 +35,8 @@ export = { position: "before", }, { - // Treat vue and composition-api as "builtin" - pattern: "(vue|@nuxtjs/composition-api)", + // Treat k6, vue and composition-api as "builtin" + pattern: "(k6|vue|@nuxtjs/composition-api)", group: "builtin", position: "before", }, diff --git a/packages/js/k6/README.md b/packages/js/k6/README.md new file mode 100644 index 00000000000..a1813e325ad --- /dev/null +++ b/packages/js/k6/README.md @@ -0,0 +1,46 @@ +# `@openverse/k6` + +**This is an internal, non-distributable package**. + +This package houses Openverse's k6 scripts. + +[Refer to k6 documentation to learn more about the tool and its capabilities](https://grafana.com/docs/k6/latest/). + +To run k6 scripts, use the just recipe: + +```shell +ov run {namespace} {scenario} [EXTRA_ARGS] +``` + +For example, to run all frontend scenarios: + +```shell +ov run frontend all +``` + +## Development tips and guidelines + +- Code is written in TypeScript. k6's + [`jslib` packages](https://grafana.com/docs/k6/latest/javascript-api/jslib/) + do not have TypeScript definitions, so you must `@ts-expect-error` those. + Rollup processes the TypeScript, and then we execute the transpiled code with + k6. +- Follow typical Openverse JavaScript code style and development procedures, but + keep in mind k6's special requirements, and the fact that + [it doesn't have a "normal" JavaScript execution environment](https://grafana.com/docs/k6/latest/javascript-api/). + - An important example of these special requirements are that k6 depends on + the scenario functions being exported from the executed test script. **This + is why the frontend `.test.ts` files all `export * from "./scenarios"`**, + otherwise the functions referenced by name in the generated scenarios would + not be available for execution by k6 from the test file. +- Test suites should be placed in namespaced directories, and then named in the + pattern `{scenario}.test.ts`. This format is mandatory for build and execution + scripts to function as intended. For example, `src/frontend/search-en.test.ts` + has the `frontend` namespace and `search-en` is the scenario name. "Scenario + name" may also refer to a _group_ of scenarios, as it does in the case of + `src/frontend/all.test.ts`, which executes all frontend scenarios. +- Use + [k6 `-e` to pass environment variables](https://grafana.com/docs/k6/latest/using-k6/environment-variables/#environment-variables) + that test code relies on to create flexible tests. For example, use this + method to write tests that can target any hostname where the target service + might be hosted. diff --git a/packages/js/k6/justfile b/packages/js/k6/justfile new file mode 100644 index 00000000000..64c59654a9f --- /dev/null +++ b/packages/js/k6/justfile @@ -0,0 +1,4 @@ +# Build and run K6 script by namespace and scenario +run namespace scenario +extra_args="": + pnpm run build --input src/{{ namespace }}/{{ scenario }}.test.ts + k6 run {{ extra_args }} ./dist/{{ namespace }}/{{ scenario }}.test.js diff --git a/packages/js/k6/package.json b/packages/js/k6/package.json new file mode 100644 index 00000000000..d6f931cfb51 --- /dev/null +++ b/packages/js/k6/package.json @@ -0,0 +1,20 @@ +{ + "name": "@openverse/k6", + "version": "0.0.0", + "description": "Openverse K6 load testing scripts", + "scripts": { + "build": "rollup --config rollup.config.ts --configPlugin typescript", + "types": "tsc -p . --noEmit", + "run": "just run" + }, + "keywords": [], + "author": "Openverse Contributors ", + "license": "MIT", + "devDependencies": { + "@rollup/plugin-typescript": "^11.1.6", + "@types/k6": "^0.53.1", + "glob": "^11.0.0", + "rollup": "^4.21.1", + "typescript": "^5.6.2" + } +} diff --git a/packages/js/k6/rollup.config.ts b/packages/js/k6/rollup.config.ts new file mode 100644 index 00000000000..78cdb005cf8 --- /dev/null +++ b/packages/js/k6/rollup.config.ts @@ -0,0 +1,33 @@ +/** + * Adapted from Apache Licensed k6 template repositories + * https://github.com/grafana/k6-rollup-example + * https://github.com/grafana/k6-template-typescript + */ +import { defineConfig } from "rollup" +import { glob } from "glob" + +import typescript from "@rollup/plugin-typescript" + +function getConfig(testFile: string) { + return defineConfig({ + input: testFile, + external: [new RegExp(/^(k6|https?:\/\/)(\/.*)?/)], + output: { + format: "es", + dir: "dist", + preserveModules: true, + preserveModulesRoot: "src", + }, + plugins: [typescript()], + }) +} + +export default defineConfig((commandLineArgs) => { + if (commandLineArgs.input) { + // --input flag passed + return getConfig(commandLineArgs.input) + } + + const tests = glob.sync("./src/**/*.test.ts") + return tests.map(getConfig) +}) diff --git a/packages/js/k6/src/frontend/all.test.ts b/packages/js/k6/src/frontend/all.test.ts new file mode 100644 index 00000000000..d5a3de506a3 --- /dev/null +++ b/packages/js/k6/src/frontend/all.test.ts @@ -0,0 +1,5 @@ +import { getOptions } from "./scenarios" + +export * from "./scenarios" + +export const options = getOptions("all") diff --git a/packages/js/k6/src/frontend/constants.ts b/packages/js/k6/src/frontend/constants.ts new file mode 100644 index 00000000000..0eae7af2b1c --- /dev/null +++ b/packages/js/k6/src/frontend/constants.ts @@ -0,0 +1,2 @@ +export const PROJECT_ID = 3713375 +export const FRONTEND_URL = __ENV.FRONTEND_URL || "https://openverse.org/" diff --git a/packages/js/k6/src/frontend/scenarios.ts b/packages/js/k6/src/frontend/scenarios.ts new file mode 100644 index 00000000000..485def35768 --- /dev/null +++ b/packages/js/k6/src/frontend/scenarios.ts @@ -0,0 +1,160 @@ +import { group } from "k6" +import exec from "k6/execution" +import http from "k6/http" + +import { getRandomWord, makeResponseFailedCheck } from "../utils.js" + +import { FRONTEND_URL, PROJECT_ID } from "./constants.js" + +import type { Options, Scenario } from "k6/options" + +const STATIC_PAGES = ["about", "sources", "privacy", "sensitive-content"] +const TEST_LOCALES = ["en", "ru", "es", "fa"] +const TEST_PARAMS = "&license=by&extension=jpg,mp3&source=flickr,jamendo" + +const localePrefix = (locale: string) => { + return locale === "en" ? "" : locale + "/" +} + +const visitUrl = (url: string, action: Action) => { + // eslint-disable-next-line import/no-named-as-default-member + const response = http.get(url, { + headers: { "User-Agent": "OpenverseLoadTesting" }, + }) + const checkResponseFailed = makeResponseFailedCheck("", url) + if (checkResponseFailed(response, action)) { + console.error(`Failed URL: ${url}`) + return 0 + } + return 1 +} + +const parseEnvLocales = (locales: string) => { + return locales ? locales.split(",") : ["en"] +} + +export function visitStaticPages() { + const locales = parseEnvLocales(__ENV.LOCALES) + console.log( + `VU: ${exec.vu.idInTest} - ITER: ${exec.vu.iterationInInstance}` + ) + for (const locale of locales) { + group(`visit static pages for locale ${locale}`, () => { + for (const page of STATIC_PAGES) { + visitUrl( + `${FRONTEND_URL}${localePrefix(locale)}${page}`, + "visitStaticPages" + ) + } + }) + } +} + +export function visitSearchPages() { + const locales = parseEnvLocales(__ENV.LOCALES) + const params = __ENV.PARAMS + const paramsString = params ? ` with params ${params}` : "" + console.log( + `VU: ${exec.vu.idInTest} - ITER: ${exec.vu.iterationInInstance}` + ) + group(`search for random word on locales ${locales}${paramsString}`, () => { + for (const MEDIA_TYPE of ["image", "audio"]) { + for (const locale of locales) { + const q = getRandomWord() + return visitUrl( + `${FRONTEND_URL}${localePrefix(locale)}search/${MEDIA_TYPE}?q=${q}${params}`, + "visitSearchPages" + ) + } + } + return undefined + }) +} + +const actions = { + visitStaticPages, + visitSearchPages, +} as const + +type Action = keyof typeof actions + +const createScenario = ( + env: Record, + funcName: Action +): Scenario => { + return { + executor: "per-vu-iterations", + env, + exec: funcName, + // k6 CLI flags do not allow override scenario options, so we need to add our own + // Ideally we would use default + // https://community.grafana.com/t/overriding-vus-individual-scenario/98923 + vus: parseFloat(__ENV.scenario_vus) || 5, + iterations: parseFloat(__ENV.scenario_iterations) || 40, + } +} + +export const SCENARIOS = { + staticPages: createScenario({ LOCALES: "en" }, "visitStaticPages"), + localeStaticPages: createScenario( + { LOCALES: TEST_LOCALES.join(",") }, + "visitStaticPages" + ), + englishSearchPages: createScenario( + { LOCALES: "en", PARAMS: "" }, + "visitSearchPages" + ), + localesSearchPages: createScenario( + { LOCALES: TEST_LOCALES.join(","), PARAMS: "" }, + "visitSearchPages" + ), + englishSearchPagesWithFilters: createScenario( + { LOCALES: "en", PARAMS: TEST_PARAMS }, + "visitSearchPages" + ), + localesSearchPagesWithFilters: createScenario( + { LOCALES: TEST_LOCALES.join(","), PARAMS: TEST_PARAMS }, + "visitSearchPages" + ), +} as const + +function getScenarios( + scenarios: (keyof typeof SCENARIOS)[] +): Record { + return scenarios.reduce( + (acc, scenario) => ({ ...acc, [scenario]: SCENARIOS[scenario] }), + {} as Record + ) +} + +export const SCENARIO_GROUPS = { + all: getScenarios([ + "staticPages", + "localeStaticPages", + "englishSearchPages", + "localesSearchPages", + "englishSearchPagesWithFilters", + "localesSearchPagesWithFilters", + ]), + "static-en": getScenarios(["staticPages"]), + "static-locales": getScenarios(["localeStaticPages"]), + "search-en": getScenarios([ + "englishSearchPages", + "englishSearchPagesWithFilters", + ]), + "search-locales": getScenarios([ + "localesSearchPages", + "localesSearchPagesWithFilters", + ]), +} satisfies Record> + +export function getOptions(group: keyof typeof SCENARIO_GROUPS): Options { + return { + scenarios: SCENARIO_GROUPS[group], + cloud: { + projectId: PROJECT_ID, + name: `Frontend ${group} ${FRONTEND_URL}`, + }, + userAgent: "OpenverseK6/1.0; https://docs.openverse.org", + } satisfies Options +} diff --git a/packages/js/k6/src/frontend/search-en.test.ts b/packages/js/k6/src/frontend/search-en.test.ts new file mode 100644 index 00000000000..c67bd3dd2ec --- /dev/null +++ b/packages/js/k6/src/frontend/search-en.test.ts @@ -0,0 +1,5 @@ +import { getOptions } from "./scenarios" + +export * from "./scenarios" + +export const options = getOptions("search-en") diff --git a/packages/js/k6/src/frontend/search-locales.test.ts b/packages/js/k6/src/frontend/search-locales.test.ts new file mode 100644 index 00000000000..66ecfac7f2e --- /dev/null +++ b/packages/js/k6/src/frontend/search-locales.test.ts @@ -0,0 +1,5 @@ +import { getOptions } from "./scenarios" + +export * from "./scenarios" + +export const options = getOptions("search-locales") diff --git a/packages/js/k6/src/frontend/static-en.test.ts b/packages/js/k6/src/frontend/static-en.test.ts new file mode 100644 index 00000000000..d5954157526 --- /dev/null +++ b/packages/js/k6/src/frontend/static-en.test.ts @@ -0,0 +1,5 @@ +import { getOptions } from "./scenarios" + +export * from "./scenarios" + +export const options = getOptions("static-en") diff --git a/packages/js/k6/src/frontend/static-locales.test.ts b/packages/js/k6/src/frontend/static-locales.test.ts new file mode 100644 index 00000000000..203894c6a09 --- /dev/null +++ b/packages/js/k6/src/frontend/static-locales.test.ts @@ -0,0 +1,5 @@ +import { getOptions } from "./scenarios" + +export * from "./scenarios" + +export const options = getOptions("static-locales") diff --git a/packages/js/k6/src/utils.ts b/packages/js/k6/src/utils.ts new file mode 100644 index 00000000000..5512e3bfe4e --- /dev/null +++ b/packages/js/k6/src/utils.ts @@ -0,0 +1,30 @@ +import { check } from "k6" +import { type Response } from "k6/http" + +// @ts-expect-error https://github.com/grafana/k6-template-typescript/issues/16 +// eslint-disable-next-line import/extensions, import/no-unresolved +import { randomItem } from "https://jslib.k6.io/k6-utils/1.2.0/index.js" + +export const SLEEP_DURATION = 0.1 + +// Use the random words list available locally, but filter any words that end with apostrophe-s +const WORDS = open("/usr/share/dict/words") + .split("\n") + .filter((w) => !w.endsWith("'s")) + +export const getRandomWord = () => randomItem(WORDS) + +export const makeResponseFailedCheck = (param: string, page: string) => { + return (response: Response, action: string) => { + const requestDetail = `${param ? `for param "${param} "` : ""}at page ${page} for ${action}` + if (check(response, { "status was 200": (r) => r.status === 200 })) { + console.log(`Checked status 200 ✓ ${requestDetail}.`) + return false + } else { + console.error( + `Request failed ⨯ ${requestDetail}: ${response.status}\n${response.body}` + ) + return true + } + } +} diff --git a/packages/js/k6/tsconfig.json b/packages/js/k6/tsconfig.json new file mode 100644 index 00000000000..7e4205a5c5c --- /dev/null +++ b/packages/js/k6/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ESNext" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + "module": "ESNext" /* Specify what module code is generated. */, + "moduleResolution": "Bundler" /* Specify how TypeScript looks up a file from a given module specifier. */, + + "noEmit": true, + "allowJs": true, + "removeComments": false, + + "paths": { + // TypeScript can already do this, but we need explicit mappings here for + // the ESLint import/resolve rule to know that k6 is actually resolveable. + "k6": ["./node_modules/@types/k6/index.d.ts"], + "k6/*": ["./node_modules/@types/k6/*"] + }, + + "strict": true, + "noImplicitAny": true, + "noImplicitThis": true, + + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + + "skipLibCheck": true + }, + "exclude": ["./dist/**"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df5699f419c..1e0de3ba8f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -342,6 +342,24 @@ importers: specifier: ^2.0.5 version: 2.0.5(@types/node@22.5.1)(@vitest/ui@2.0.5)(jsdom@24.1.0)(terser@5.31.0) + packages/js/k6: + devDependencies: + '@rollup/plugin-typescript': + specifier: ^11.1.6 + version: 11.1.6(rollup@4.21.2)(tslib@2.6.2)(typescript@5.6.2) + '@types/k6': + specifier: ^0.53.1 + version: 0.53.1 + glob: + specifier: ^11.0.0 + version: 11.0.0 + rollup: + specifier: ^4.21.1 + version: 4.21.2 + typescript: + specifier: ^5.6.2 + version: 5.6.2 + packages: '@aashutoshrathi/word-wrap@1.2.6': @@ -2808,6 +2826,9 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + '@types/k6@0.53.1': + resolution: {integrity: sha512-Mm1+7kESDNssyBiMV+5MVAJKwpJbj1Y3X1YkCDlaiPbjuc8DeaPtQFBv3pnip/0ZjM9P/P/aDNvd4BvQT0lQDg==} + '@types/keyv@3.1.4': resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} @@ -7579,6 +7600,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@5.6.2: + resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==} + engines: {node: '>=14.17'} + hasBin: true + ufo@1.5.4: resolution: {integrity: sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==} @@ -10620,6 +10646,15 @@ snapshots: rollup: 4.21.2 tslib: 2.6.2 + '@rollup/plugin-typescript@11.1.6(rollup@4.21.2)(tslib@2.6.2)(typescript@5.6.2)': + dependencies: + '@rollup/pluginutils': 5.1.0(rollup@4.21.2) + resolve: 1.22.8 + typescript: 5.6.2 + optionalDependencies: + rollup: 4.21.2 + tslib: 2.6.2 + '@rollup/plugin-yaml@4.1.2(rollup@4.21.2)': dependencies: '@rollup/pluginutils': 5.1.0(rollup@4.21.2) @@ -11351,6 +11386,8 @@ snapshots: '@types/json5@0.0.29': {} + '@types/k6@0.53.1': {} + '@types/keyv@3.1.4': dependencies: '@types/node': 22.5.1 @@ -16982,6 +17019,8 @@ snapshots: typescript@5.5.4: {} + typescript@5.6.2: {} + ufo@1.5.4: {} ultrahtml@1.5.3: {} diff --git a/utilities/load_testing/.env.sh.template b/utilities/load_testing/.env.sh.template deleted file mode 100644 index 2dda0a78118..00000000000 --- a/utilities/load_testing/.env.sh.template +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/env sh - -export ACCESS_TOKEN="" diff --git a/utilities/load_testing/.gitignore b/utilities/load_testing/.gitignore deleted file mode 100644 index 3814618cd66..00000000000 --- a/utilities/load_testing/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.env.sh -output.html diff --git a/utilities/load_testing/Dockerfile b/utilities/load_testing/Dockerfile deleted file mode 100644 index 633e9b246ef..00000000000 --- a/utilities/load_testing/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# syntax=docker/dockerfile:1 - -###### -# K6 # -###### - -FROM docker.io/grafana/k6:0.51.0 AS k6 - -############### -# Observation # -############### - -FROM docker.io/python:3.11-slim AS lt - -LABEL org.opencontainers.image.source="https://github.com/WordPress/openverse" - -# Copy `k6` binary -COPY --from=k6 /usr/bin/k6 /usr/bin/k6 - -# Install system packages needed by the project -# - wamerican: American words dictionary -# - apache2-utils: Apache HTTP server benchmarking tool `ab` -# - gnupg2: TODO why? -RUN apt-get update \ - && apt-get install -yqq --no-install-recommends \ - wamerican \ - apache2-utils gnupg2 \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -ENTRYPOINT ["k6"] diff --git a/utilities/load_testing/README.md b/utilities/load_testing/README.md deleted file mode 100644 index 4a1f71b34ff..00000000000 --- a/utilities/load_testing/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Load testing - -This directory contains a collection of locust files for load testing the -Openverse application. Currently, only an API load test exists but in the future -we can also create load tests for the frontend or any other service we stand up. -These exist in this repository rather than one of the service specific ones for -this reason. We can reuse the base docker image we create for all our load -testing scripts. - -**Note:** Production is throttled, so these scripts cannot be used as a shortcut -to DDoS our production stack. - -## Running the tests - -### Setup - -#### API - -The API load tests assume that you will be using an API key as it is impossible -to make unauthenticated requests large enough (`page_size=500`) to facilitate -easy flooding. Likewise, unless throttling is temporarily disabled in the -environment being tested, you'll probably want to exempt the API key from -throttling by adding it to the -[API Key throttle exemption](https://github.com/WordPress/openverse-api/blob/c09fd7e16a8eb104c311e8d4f0da08238570067c/api/catalog/api/utils/throttle.py#L77). -Please ping `@WordPress/openverse-maintainers` for help doing this in live -environments as it requires adding the key to a set in that environment's Redis -cache. - -To make the API key accessible to the load testing script, copy the -`.env.sh.template` file and name it `.env.sh` and fill in the `ACCESS_TOKEN` -variable. - -#### Frontend - -By default, the frontend load tests run against `https://openverse.org`. It is -important to disable the Cloudflare cache and WAF security settings for the -duration of the test to test the actual performance of the application instead -of testing Cloudflare cache or getting blocked by WAF. By default, the tests -request only the English static pages. You can specify other scenarios to run, -with `ov just load_testing/k6 all` being the most comprehensive test that -requests static and search pages in English. The `locales` tests request pages -in 3 other locales. - -### Running - -All load tests are accessible through `just` scripts in this directory's -`justfile`. - -To run API load tests against a local API instance, use -`ov just load_testing/api`. You can optionally specify a host (the default is to -point to local). For example -`ov just load_testing/api https://api-staging.openverse.org` will run the load -tests against the staging API. - -To run the frontend load tests, use `ov just load_testing/k6-frontend`. You can -optionally specify the scenarios to run, for example, -`ov just load_testing/k6-frontend all`. To specify a host (the default is to -point to the `openverse.org`), set the `FRONTEND_URL` environment variable. For -example, -`ov env FRONTEND_URL=https://staging.openverse.org just load_testing/k6-frontend` -will run the load tests against the staging frontend. diff --git a/utilities/load_testing/api.sh b/utilities/load_testing/api.sh deleted file mode 100755 index e0640307c88..00000000000 --- a/utilities/load_testing/api.sh +++ /dev/null @@ -1,32 +0,0 @@ -#! /usr/bin/env bash - -set -e - -# shellcheck source=/dev/null -source ./.env.sh || (echo "Please create a $(.env.sh) file based on the $(.env.sh.template) file." >/dev/stderr && false) - -host=$1 - -auth_header="Authorization: Bearer $ACCESS_TOKEN" - -concurrency=4 - -requests=100 - -q=$(./get_word.sh) - -set -x - -ab \ - -w \ - -v 3 \ - -c $concurrency \ - -n $requests \ - -T "application/json" \ - -H "$auth_header" \ - "$host/v1/images/?$q&page_size=500" >output.html - -set +x - -# fix root permissions -chown 1000:1000 output.html diff --git a/utilities/load_testing/get_word.sh b/utilities/load_testing/get_word.sh deleted file mode 100755 index 3b73619270f..00000000000 --- a/utilities/load_testing/get_word.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/env sh - -python -c "from urllib import parse; print(parse.urlencode({'q': \"$(shuf -n1 /usr/share/dict/words)\"}), end='')" diff --git a/utilities/load_testing/justfile b/utilities/load_testing/justfile deleted file mode 100644 index d2fcad217da..00000000000 --- a/utilities/load_testing/justfile +++ /dev/null @@ -1,38 +0,0 @@ -set dotenv-load := true - -COLOR := "\\033[0;32m" -NO_COLOR := "\\033[0m" - -# Show all available recipes -@_default: - printf "\n{{ COLOR }}# Load testing (path: \`api/\`)\n" - printf "============================={{ NO_COLOR }}\n" - just --list --unsorted - -DOCKER_HOST := if os() == "macos" { "host.docker.internal" } else { "172.17.0.1" } - -build: - docker build . -t openverse-load_testing:latest - -run script host: build - docker run --rm -it -v {{ invocation_directory() }}:/app openverse-load-testing:latest ./{{ script }}.sh {{ host }} - -api host=("http://" + DOCKER_HOST + ":50280"): - @just run api {{ host }} - -# Run the specified script in a load testing container -k6 script="main.js" +extra_args="": build - docker run \ - --rm \ - ${ACCESS_TOKEN:+ --env ACCESS_TOKEN=$ACCESS_TOKEN} \ - ${API_URL:+ --env API_URL=$API_URL} \ - ${FRONTEND_URL:+ --env FRONTEND_URL=$FRONTEND_URL} \ - --volume {{ invocation_directory() }}/k6:/app \ - {{ extra_args }} \ - openverse-load_testing:latest \ - run {{ script }} - - -# Run the frontend script for one of the scenarios (static-en, static-locales, search-en, search-locales, all) -k6-frontend scenarios="static-en": - @just k6 frontend.js --env SCENARIOS={{scenarios}} diff --git a/utilities/load_testing/k6/frontend.js b/utilities/load_testing/k6/frontend.js deleted file mode 100644 index e1aaaa56b84..00000000000 --- a/utilities/load_testing/k6/frontend.js +++ /dev/null @@ -1,139 +0,0 @@ -import { group } from "k6" -import exec from "k6/execution" -import http from "k6/http" -import { - FRONTEND_URL, - getRandomWord, - makeResponseFailedCheck, -} from "./utils.js" - -const STATIC_PAGES = ["about", "sources", "privacy", "sensitive-content"] -const TEST_LOCALES = ["en", "ru", "es", "fa"] -const TEST_PARAMS = "&license=by&extension=jpg,mp3&source=flickr,jamendo" - -const localePrefix = (locale) => { - return locale === "en" ? "" : locale + "/" -} - -const visitUrl = (url, locale, action) => { - const response = http.get(url, { - headers: { "User-Agent": "OpenverseLoadTesting" }, - }) - const checkResponseFailed = makeResponseFailedCheck("", url) - if (checkResponseFailed(response, action)) { - console.error(`Failed URL: ${url}`) - return 0 - } -} - -const parseEnvLocales = (locales) => { - return locales ? locales.split(",") : ["en"] -} - -export function visitStaticPages() { - const locales = parseEnvLocales(__ENV.LOCALES) - console.log( - `VU: ${exec.vu.idInTest} - ITER: ${exec.vu.iterationInInstance}` - ) - for (let locale of locales) { - group(`visit static pages for locale ${locale}`, () => { - for (let page of STATIC_PAGES) { - visitUrl( - `${FRONTEND_URL}${localePrefix(locale)}${page}`, - locale, - "visitPage" - ) - } - }) - } -} - -export function visitSearchPages() { - let locales = parseEnvLocales(__ENV.LOCALES) - let params = __ENV.PARAMS - const paramsString = params ? ` with params ${params}` : "" - console.log( - `VU: ${exec.vu.idInTest} - ITER: ${exec.vu.iterationInInstance}` - ) - group(`search for random word on locales ${locales}${paramsString}`, () => { - for (let MEDIA_TYPE of ["image", "audio"]) { - for (let locale of locales) { - let q = getRandomWord() - return visitUrl( - `${FRONTEND_URL}${localePrefix(locale)}search/${MEDIA_TYPE}?q=${q}${params}`, - locale, - "visitSearchPage" - ) - } - } - }) -} - -const createScenario = (env, funcName) => { - return { - executor: "per-vu-iterations", - env, - exec: funcName, - vus: 5, - iterations: 40, - } -} - -const optionToScenario = { - all: [ - "staticPages", - "localeStaticPages", - "englishSearchPages", - "localesSearchPages", - "englishSearchPagesWithFilters", - "localesSearchPagesWithFilters", - ], - "static-en": ["staticPages"], - "static-locales": ["localeStaticPages"], - "search-en": ["englishSearchPages", "englishSearchPagesWithFilters"], - "search-locales": ["localesSearchPages", "localesSearchPagesWithFilters"], -} -const getScenariosToRun = () => { - let scenariosToRun = __ENV.SCENARIOS - - if ( - !scenariosToRun || - !Object.keys(optionToScenario).includes(scenariosToRun) - ) { - scenariosToRun = "static-en" - } - const allScenarios = { - staticPages: createScenario({ LOCALES: "en" }, "visitStaticPages"), - localeStaticPages: createScenario( - { LOCALES: TEST_LOCALES.join(",") }, - "visitStaticPages" - ), - englishSearchPages: createScenario( - { LOCALES: "en", PARAMS: "" }, - "visitSearchPages" - ), - localesSearchPages: createScenario( - { LOCALES: TEST_LOCALES.join(","), PARAMS: "" }, - "visitSearchPages" - ), - englishSearchPagesWithFilters: createScenario( - { LOCALES: "en", PARAMS: TEST_PARAMS }, - "visitSearchPages" - ), - localesSearchPagesWithFilters: createScenario( - { LOCALES: TEST_LOCALES.join(","), PARAMS: TEST_PARAMS }, - "visitSearchPages" - ), - } - - return Object.keys(allScenarios).reduce((acc, key) => { - if (optionToScenario[scenariosToRun].includes(key)) { - acc[key] = allScenarios[key] - } - return acc - }, {}) -} - -export const options = { - scenarios: getScenariosToRun(), -} diff --git a/utilities/load_testing/k6/main.js b/utilities/load_testing/k6/main.js deleted file mode 100644 index e9243f7aece..00000000000 --- a/utilities/load_testing/k6/main.js +++ /dev/null @@ -1,70 +0,0 @@ -import { group } from "k6" -import { searchBy } from "./search.js" -import { getProvider, getRandomWord } from "./utils.js" - -const createScenario = (mediaType, pageSize, funcName) => { - return { - executor: "per-vu-iterations", - env: { - MEDIA_TYPE: mediaType, - PAGE_SIZE: pageSize, - }, - exec: funcName, - vus: 5, - iterations: 5, - } -} - -export const options = { - scenarios: { - random_word_image_page_20: createScenario( - "images", - "20", - "searchByRandomWord" - ), - random_word_audio_page_20: createScenario( - "audio", - "20", - "searchByRandomWord" - ), - random_word_image_page_500: createScenario( - "images", - "500", - "searchByRandomWord" - ), - random_word_audio_page_500: createScenario( - "audio", - "500", - "searchByRandomWord" - ), - provider_image_page_20: createScenario("image", "20", "searchByProvider"), - provider_image_page_500: createScenario("image", "500", "searchByProvider"), - provider_audio_page_20: createScenario("audio", "20", "searchByProvider"), - provider_audio_page_500: createScenario("audio", "500", "searchByProvider"), - }, -} - -const searchByField = (paramFunc, followLinks = false) => { - const MEDIA_TYPE = __ENV.MEDIA_TYPE - const PAGE_SIZE = __ENV.PAGE_SIZE - console.log(`VU: ${__VU} - ITER: ${__ITER}`) - const param = paramFunc(MEDIA_TYPE) - const depth = followLinks ? "Deep" : "Shallow" - - group( - `${depth} ${MEDIA_TYPE} search of ${PAGE_SIZE} items (using '${param}')`, - () => { - let page = 1 - let page_count = 1 - while (page <= page_count) { - page_count = searchBy(param, page, MEDIA_TYPE, PAGE_SIZE, followLinks) - page++ - } - } - ) -} - -export const searchByRandomWord = () => - searchByField(() => `q=${getRandomWord()}`, true) -export const searchByProvider = () => - searchByField((media_type) => `source=${getProvider(media_type)}`, false) diff --git a/utilities/load_testing/k6/observation.js b/utilities/load_testing/k6/observation.js deleted file mode 100644 index 97896d42f6a..00000000000 --- a/utilities/load_testing/k6/observation.js +++ /dev/null @@ -1,88 +0,0 @@ -/** - * This module defines tests to run on the API to observe that everything is - * working normally while we migrate Redis. These are not load tests. - */ - -import { sleep } from "k6" -import http from "k6/http" -import { Counter, Trend } from "k6/metrics" - -import { API_URL, REQUEST_HEADERS } from "./utils.js" - -const count200 = new Counter("count_200") -const count424 = new Counter("count_424") -const count429 = new Counter("count_429") -const count4xx = new Counter("count_4xx") -const count5xx = new Counter("count_5xx") -const countNoneUsage = new Counter("count_none_usage") - -const trendBurst = new Trend("trend_usage") -const trendSustained = new Trend("trend_sustained") - -const createScenario = (env, exec) => { - return { - executor: "constant-vus", - duration: __ENV.DURATION || "5m", - vus: 1, - env, - exec, - } -} - -export const options = { - scenarios: { - imageStats: createScenario({ MEDIA_TYPE: "images" }, "stats"), - audioStats: createScenario({ MEDIA_TYPE: "audio" }, "stats"), - imageSearch: createScenario({ MEDIA_TYPE: "images" }, "search"), - audioSearch: createScenario({ MEDIA_TYPE: "audio" }, "search"), - authInfo: createScenario({}, "auth"), - }, -} - -const updateMetrics = (response) => { - if (response.status === 200) { - count200.add(1) - } else if (response.status === 424) { - count424.add(1) - } else if (response.status === 429) { - count429.add(1) - } else if (response.status >= 400 && response.status < 500) { - count4xx.add(1) - } else if (response.status >= 500 && response.status < 600) { - count5xx.add(1) - } -} - -export const auth = () => { - let response = http.get(`${API_URL}rate_limit/`, { headers: REQUEST_HEADERS }) - updateMetrics(response) - if (response.status === 200 || response.status === 424) { - const burst = JSON.parse(response.body).requests_this_minute - const sustained = JSON.parse(response.body).requests_today - if (burst === null || sustained === null) { - countNoneUsage.add(1) - } else { - trendBurst.add(burst) - trendSustained.add(sustained) - } - } - sleep(3) // seconds -} - -export const stats = () => { - const mediaType = __ENV.MEDIA_TYPE - let response = http.get(`${API_URL}${mediaType}/stats`, { - headers: REQUEST_HEADERS, - }) - updateMetrics(response) - sleep(3) // seconds -} - -export const search = () => { - const mediaType = __ENV.MEDIA_TYPE - let response = http.get(`${API_URL}${mediaType}/?q=cat`, { - headers: REQUEST_HEADERS, - }) - updateMetrics(response) - sleep(3) // seconds -} diff --git a/utilities/load_testing/k6/search.js b/utilities/load_testing/k6/search.js deleted file mode 100644 index bba009e2e4d..00000000000 --- a/utilities/load_testing/k6/search.js +++ /dev/null @@ -1,53 +0,0 @@ -import http from "k6/http" -import { group, sleep } from "k6" -import { - API_URL, - getUrlBatch, - makeResponseFailedCheck, - REQUEST_HEADERS, - SLEEP_DURATION, -} from "./utils.js" - -export const searchBy = (param, page, media_type, page_size, followLinks) => { - let url = `${API_URL}${media_type}/?${param}&page=${page}&page_size=${page_size}&filter_dead=false` - const response = http.get(url, { headers: REQUEST_HEADERS }) - - const checkResponseFailed = makeResponseFailedCheck(param, page) - - if (checkResponseFailed(response, "search")) { - console.error(`Failed URL: ${url}`) - return 0 - } - - const parsedResp = response.json() - const pageCount = parsedResp["page_count"] - const detailUrls = parsedResp["results"].map((i) => i.detail_url) - const relatedUrls = parsedResp["results"].map((i) => i.related_url) - - // Don't view details/related if not requested - if (!followLinks) { - return pageCount - } - - group("Details requests", () => { - console.info( - `Requesting all ${media_type} details from "${param}" at page ${page}` - ) - const responses = http.batch(getUrlBatch(detailUrls)) - responses.map((r) => checkResponseFailed(r, "details")) - }) - - sleep(SLEEP_DURATION) - - group("Related requests", () => { - console.info( - `Requesting all ${media_type} related from "${param}" at page ${page}` - ) - const responses = http.batch(getUrlBatch(relatedUrls, "related_url")) - responses.map((r) => checkResponseFailed(r, "related")) - }) - - sleep(SLEEP_DURATION) - - return pageCount -} diff --git a/utilities/load_testing/k6/utils.js b/utilities/load_testing/k6/utils.js deleted file mode 100644 index 966706a5099..00000000000 --- a/utilities/load_testing/k6/utils.js +++ /dev/null @@ -1,48 +0,0 @@ -import { check } from "k6" -import http from "k6/http" -import { randomItem } from "https://jslib.k6.io/k6-utils/1.2.0/index.js" - -export const API_URL = __ENV.API_URL || "https://api-staging.openverse.org/v1/" -export const FRONTEND_URL = __ENV.FRONTEND_URL || "https://openverse.org/" - -export const SLEEP_DURATION = 0.1 -// Use the random words list available locally, but filter any words that end with apostrophe-s -const WORDS = open("/usr/share/dict/words") - .split("\n") - .filter((w) => !w.endsWith("'s")) - -export const getRandomWord = () => randomItem(WORDS) - -export const getProvider = (media_type) => { - let url = `${API_URL}${media_type}/stats` - const response = http.get(url, { headers: REQUEST_HEADERS }) - let providers = JSON.parse(response.body) - return randomItem(providers).source_name -} - -export const REQUEST_HEADERS = { - Authorization: `Bearer ${__ENV.ACCESS_TOKEN}`, - "User-Agent": "k6", -} - -export const getUrlBatch = (urls, type = "detail_url") => { - return urls.map((u) => { - const params = { headers: REQUEST_HEADERS, tags: { name: type } } - return ["GET", u, null, params] - }) -} - -export const makeResponseFailedCheck = (param, page) => { - return (response, action) => { - const requestDetail = `${param ? `for param "${param} "` : ""}at page ${page} for ${action}` - if (check(response, { "status was 200": (r) => r.status === 200 })) { - console.log(`Checked status 200 ✓ ${requestDetail}.`) - return false - } else { - console.error( - `Request failed ⨯ ${requestDetail}: ${response.status}\n${response.body}` - ) - return true - } - } -}