Refactor k6 development environment to prepare for regular runs in CD (…

…#4903)

* Add k6 to ov

* Improve Openverse k6 devex with TS and ESLint

* Fix args passing and allow overriding scenario options

* Fix broken eslint import config

docker/dev_env/Dockerfile

@@ -37,6 +37,7 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}"
 # - nodejs: language runtime (includes npm but not Corepack)
 # - docker*: used to interact with host Docker socket
 # - postgresql*: required to connect to PostgreSQL databases
+# - k6, words: used for load testing
 #
 # pipx dependencies:
 # - httpie: CLI HTTP client
@@ -50,6 +51,7 @@ ENV PATH="${PNPM_BIN}:${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}"
 RUN mkdir /pipx \
   && dnf -y install dnf-plugins-core \
   && dnf -y config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \
+  && dnf -y install https://dl.k6.io/rpm/repo.rpm \
   && dnf -y install \
     git \
     g++ \
@@ -63,6 +65,7 @@ RUN mkdir /pipx \
     docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
     unzip \
     postgresql postgresql-devel \
+    k6 words \
   && dnf clean all \
   && pipx install --global \
     httpie \

frontend/package.json

@@ -41,7 +41,7 @@
     "test:storybook:local": "playwright test -c test/storybook",
     "test:storybook:debug": "PWDEBUG=1 pnpm test:storybook:local",
     "test:storybook:gen": "playwright codegen localhost:54000/",
-    "types": "vue-tsc -p .",
+    "types": "pnpm run prepare:nuxt && vue-tsc -p .",
     "i18n": "pnpm i18n:create-locales-list && pnpm i18n:get-translations && pnpm i18n:update-locales",
     "i18n:en": "pnpm i18n:get-translations --en-only",
     "i18n:copy-test-locales": "cp test/locales/**.json src/locales/ && mv src/locales/valid-locales.json src/locales/scripts/valid-locales.json",

justfile

@@ -357,11 +357,15 @@ p package script +args="":
 
 # Run eslint with --fix and default file selection enabled; used to enable easy file overriding whilst retaining the defaults when running --all-files
 eslint *files="frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json":
-    just p '@openverse/eslint-plugin' run build
+    just p '@openverse/eslint-plugin' build
     pnpm exec eslint \
         --ext .js,.ts,.vue,.json,.json5 \
         --ignore-path .gitignore \
         --ignore-path .eslintignore \
         --max-warnings=0 \
         --fix \
         {{ files }}
+
+# Alias for `just packages/js/k6/run` or `just p k6 run`
+@k6 *args:
+    just packages/js/k6/run {{ args }}

packages/js/eslint-plugin/src/configs/import.ts

@@ -35,8 +35,8 @@ export = {
             position: "before",
           },
           {
-            // Treat vue and composition-api as "builtin"
-            pattern: "(vue|@nuxtjs/composition-api)",
+            // Treat k6, vue and composition-api as "builtin"
+            pattern: "(k6|vue|@nuxtjs/composition-api)",
             group: "builtin",
             position: "before",
           },

packages/js/k6/README.md

@@ -0,0 +1,46 @@
+# `@openverse/k6`
+
+**This is an internal, non-distributable package**.
+
+This package houses Openverse's k6 scripts.
+
+[Refer to k6 documentation to learn more about the tool and its capabilities](https://grafana.com/docs/k6/latest/).
+
+To run k6 scripts, use the just recipe:
+
+```shell
+ov run {namespace} {scenario} [EXTRA_ARGS]
+```
+
+For example, to run all frontend scenarios:
+
+```shell
+ov run frontend all
+```
+
+## Development tips and guidelines
+
+- Code is written in TypeScript. k6's
+  [`jslib` packages](https://grafana.com/docs/k6/latest/javascript-api/jslib/)
+  do not have TypeScript definitions, so you must `@ts-expect-error` those.
+  Rollup processes the TypeScript, and then we execute the transpiled code with
+  k6.
+- Follow typical Openverse JavaScript code style and development procedures, but
+  keep in mind k6's special requirements, and the fact that
+  [it doesn't have a "normal" JavaScript execution environment](https://grafana.com/docs/k6/latest/javascript-api/).
+  - An important example of these special requirements are that k6 depends on
+    the scenario functions being exported from the executed test script. **This
+    is why the frontend `.test.ts` files all `export * from "./scenarios"`**,
+    otherwise the functions referenced by name in the generated scenarios would
+    not be available for execution by k6 from the test file.
+- Test suites should be placed in namespaced directories, and then named in the
+  pattern `{scenario}.test.ts`. This format is mandatory for build and execution
+  scripts to function as intended. For example, `src/frontend/search-en.test.ts`
+  has the `frontend` namespace and `search-en` is the scenario name. "Scenario
+  name" may also refer to a _group_ of scenarios, as it does in the case of
+  `src/frontend/all.test.ts`, which executes all frontend scenarios.
+- Use
+  [k6 `-e` to pass environment variables](https://grafana.com/docs/k6/latest/using-k6/environment-variables/#environment-variables)
+  that test code relies on to create flexible tests. For example, use this
+  method to write tests that can target any hostname where the target service
+  might be hosted.

packages/js/k6/justfile

@@ -0,0 +1,4 @@
+# Build and run K6 script by namespace and scenario
+run namespace scenario +extra_args="":
+    pnpm run build --input src/{{ namespace }}/{{ scenario }}.test.ts
+    k6 run {{ extra_args }} ./dist/{{ namespace }}/{{ scenario }}.test.js

packages/js/k6/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@openverse/k6",
+  "version": "0.0.0",
+  "description": "Openverse K6 load testing scripts",
+  "scripts": {
+    "build": "rollup --config rollup.config.ts --configPlugin typescript",
+    "types": "tsc -p . --noEmit",
+    "run": "just run"
+  },
+  "keywords": [],
+  "author": "Openverse Contributors <openverse@wordpress.org>",
+  "license": "MIT",
+  "devDependencies": {
+    "@rollup/plugin-typescript": "^11.1.6",
+    "@types/k6": "^0.53.1",
+    "glob": "^11.0.0",
+    "rollup": "^4.21.1",
+    "typescript": "^5.6.2"
+  }
+}

packages/js/k6/rollup.config.ts

@@ -0,0 +1,33 @@
+/**
+ * Adapted from Apache Licensed k6 template repositories
+ * https://github.com/grafana/k6-rollup-example
+ * https://github.com/grafana/k6-template-typescript
+ */
+import { defineConfig } from "rollup"
+import { glob } from "glob"
+
+import typescript from "@rollup/plugin-typescript"
+
+function getConfig(testFile: string) {
+  return defineConfig({
+    input: testFile,
+    external: [new RegExp(/^(k6|https?:\/\/)(\/.*)?/)],
+    output: {
+      format: "es",
+      dir: "dist",
+      preserveModules: true,
+      preserveModulesRoot: "src",
+    },
+    plugins: [typescript()],
+  })
+}
+
+export default defineConfig((commandLineArgs) => {
+  if (commandLineArgs.input) {
+    // --input flag passed
+    return getConfig(commandLineArgs.input)
+  }
+
+  const tests = glob.sync("./src/**/*.test.ts")
+  return tests.map(getConfig)
+})

packages/js/k6/src/frontend/all.test.ts

@@ -0,0 +1,5 @@
+import { getOptions } from "./scenarios"
+
+export * from "./scenarios"
+
+export const options = getOptions("all")

packages/js/k6/src/frontend/constants.ts

@@ -0,0 +1,2 @@
+export const PROJECT_ID = 3713375
+export const FRONTEND_URL = __ENV.FRONTEND_URL || "https://openverse.org/"

packages/js/k6/src/frontend/scenarios.ts

@@ -0,0 +1,160 @@
+import { group } from "k6"
+import exec from "k6/execution"
+import http from "k6/http"
+
+import { getRandomWord, makeResponseFailedCheck } from "../utils.js"
+
+import { FRONTEND_URL, PROJECT_ID } from "./constants.js"
+
+import type { Options, Scenario } from "k6/options"
+
+const STATIC_PAGES = ["about", "sources", "privacy", "sensitive-content"]
+const TEST_LOCALES = ["en", "ru", "es", "fa"]
+const TEST_PARAMS = "&license=by&extension=jpg,mp3&source=flickr,jamendo"
+
+const localePrefix = (locale: string) => {
+  return locale === "en" ? "" : locale + "/"
+}
+
+const visitUrl = (url: string, action: Action) => {
+  // eslint-disable-next-line import/no-named-as-default-member
+  const response = http.get(url, {
+    headers: { "User-Agent": "OpenverseLoadTesting" },
+  })
+  const checkResponseFailed = makeResponseFailedCheck("", url)
+  if (checkResponseFailed(response, action)) {
+    console.error(`Failed URL: ${url}`)
+    return 0
+  }
+  return 1
+}
+
+const parseEnvLocales = (locales: string) => {
+  return locales ? locales.split(",") : ["en"]
+}
+
+export function visitStaticPages() {
+  const locales = parseEnvLocales(__ENV.LOCALES)
+  console.log(
+    `VU: ${exec.vu.idInTest}  -  ITER: ${exec.vu.iterationInInstance}`
+  )
+  for (const locale of locales) {
+    group(`visit static pages for locale ${locale}`, () => {
+      for (const page of STATIC_PAGES) {
+        visitUrl(
+          `${FRONTEND_URL}${localePrefix(locale)}${page}`,
+          "visitStaticPages"
+        )
+      }
+    })
+  }
+}
+
+export function visitSearchPages() {
+  const locales = parseEnvLocales(__ENV.LOCALES)
+  const params = __ENV.PARAMS
+  const paramsString = params ? ` with params ${params}` : ""
+  console.log(
+    `VU: ${exec.vu.idInTest}  -  ITER: ${exec.vu.iterationInInstance}`
+  )
+  group(`search for random word on locales ${locales}${paramsString}`, () => {
+    for (const MEDIA_TYPE of ["image", "audio"]) {
+      for (const locale of locales) {
+        const q = getRandomWord()
+        return visitUrl(
+          `${FRONTEND_URL}${localePrefix(locale)}search/${MEDIA_TYPE}?q=${q}${params}`,
+          "visitSearchPages"
+        )
+      }
+    }
+    return undefined
+  })
+}
+
+const actions = {
+  visitStaticPages,
+  visitSearchPages,
+} as const
+
+type Action = keyof typeof actions
+
+const createScenario = (
+  env: Record<string, string>,
+  funcName: Action
+): Scenario => {
+  return {
+    executor: "per-vu-iterations",
+    env,
+    exec: funcName,
+    // k6 CLI flags do not allow override scenario options, so we need to add our own
+    // Ideally we would use default
+    // https://community.grafana.com/t/overriding-vus-individual-scenario/98923
+    vus: parseFloat(__ENV.scenario_vus) || 5,
+    iterations: parseFloat(__ENV.scenario_iterations) || 40,
+  }
+}
+
+export const SCENARIOS = {
+  staticPages: createScenario({ LOCALES: "en" }, "visitStaticPages"),
+  localeStaticPages: createScenario(
+    { LOCALES: TEST_LOCALES.join(",") },
+    "visitStaticPages"
+  ),
+  englishSearchPages: createScenario(
+    { LOCALES: "en", PARAMS: "" },
+    "visitSearchPages"
+  ),
+  localesSearchPages: createScenario(
+    { LOCALES: TEST_LOCALES.join(","), PARAMS: "" },
+    "visitSearchPages"
+  ),
+  englishSearchPagesWithFilters: createScenario(
+    { LOCALES: "en", PARAMS: TEST_PARAMS },
+    "visitSearchPages"
+  ),
+  localesSearchPagesWithFilters: createScenario(
+    { LOCALES: TEST_LOCALES.join(","), PARAMS: TEST_PARAMS },
+    "visitSearchPages"
+  ),
+} as const
+
+function getScenarios(
+  scenarios: (keyof typeof SCENARIOS)[]
+): Record<string, Scenario> {
+  return scenarios.reduce(
+    (acc, scenario) => ({ ...acc, [scenario]: SCENARIOS[scenario] }),
+    {} as Record<string, Scenario>
+  )
+}
+
+export const SCENARIO_GROUPS = {
+  all: getScenarios([
+    "staticPages",
+    "localeStaticPages",
+    "englishSearchPages",
+    "localesSearchPages",
+    "englishSearchPagesWithFilters",
+    "localesSearchPagesWithFilters",
+  ]),
+  "static-en": getScenarios(["staticPages"]),
+  "static-locales": getScenarios(["localeStaticPages"]),
+  "search-en": getScenarios([
+    "englishSearchPages",
+    "englishSearchPagesWithFilters",
+  ]),
+  "search-locales": getScenarios([
+    "localesSearchPages",
+    "localesSearchPagesWithFilters",
+  ]),
+} satisfies Record<string, Record<string, Scenario>>
+
+export function getOptions(group: keyof typeof SCENARIO_GROUPS): Options {
+  return {
+    scenarios: SCENARIO_GROUPS[group],
+    cloud: {
+      projectId: PROJECT_ID,
+      name: `Frontend ${group} ${FRONTEND_URL}`,
+    },
+    userAgent: "OpenverseK6/1.0; https://docs.openverse.org",
+  } satisfies Options
+}

packages/js/k6/src/frontend/search-en.test.ts

@@ -0,0 +1,5 @@
+import { getOptions } from "./scenarios"
+
+export * from "./scenarios"
+
+export const options = getOptions("search-en")

packages/js/k6/src/frontend/search-locales.test.ts

@@ -0,0 +1,5 @@
+import { getOptions } from "./scenarios"
+
+export * from "./scenarios"
+
+export const options = getOptions("search-locales")

packages/js/k6/src/frontend/static-en.test.ts

@@ -0,0 +1,5 @@
+import { getOptions } from "./scenarios"
+
+export * from "./scenarios"
+
+export const options = getOptions("static-en")

packages/js/k6/src/frontend/static-locales.test.ts

@@ -0,0 +1,5 @@
+import { getOptions } from "./scenarios"
+
+export * from "./scenarios"
+
+export const options = getOptions("static-locales")

packages/js/k6/src/utils.ts

@@ -0,0 +1,30 @@
+import { check } from "k6"
+import { type Response } from "k6/http"
+
+// @ts-expect-error https://github.com/grafana/k6-template-typescript/issues/16
+// eslint-disable-next-line import/extensions, import/no-unresolved
+import { randomItem } from "https://jslib.k6.io/k6-utils/1.2.0/index.js"
+
+export const SLEEP_DURATION = 0.1
+
+// Use the random words list available locally, but filter any words that end with apostrophe-s
+const WORDS = open("/usr/share/dict/words")
+  .split("\n")
+  .filter((w) => !w.endsWith("'s"))
+
+export const getRandomWord = () => randomItem(WORDS)
+
+export const makeResponseFailedCheck = (param: string, page: string) => {
+  return (response: Response, action: string) => {
+    const requestDetail = `${param ? `for param "${param} "` : ""}at page ${page} for ${action}`
+    if (check(response, { "status was 200": (r) => r.status === 200 })) {
+      console.log(`Checked status 200 ✓ ${requestDetail}.`)
+      return false
+    } else {
+      console.error(
+        `Request failed ⨯ ${requestDetail}: ${response.status}\n${response.body}`
+      )
+      return true
+    }
+  }
+}