From d9c8ac8bbce28f9935f0c259624249bde5c3a6c9 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 19:28:31 +0100
Subject: [PATCH 01/19] loadtest

---
 .../federation-example.memtest.ts             |  22 +++
 internal/e2e/src/tenv.ts                      |  17 ++-
 internal/memtest/package.json                 |  14 ++
 internal/memtest/src/index.ts                 |   1 +
 internal/memtest/src/loadtest-script.ts       |  32 +++++
 internal/memtest/src/loadtest.ts              | 130 ++++++++++++++++++
 internal/proc/src/index.ts                    |   6 +
 package.json                                  |   3 +-
 tsconfig.json                                 |   1 +
 vitest.projects.ts                            |   9 ++
 yarn.lock                                     |  32 +++++
 11 files changed, 259 insertions(+), 8 deletions(-)
 create mode 100644 e2e/federation-example/federation-example.memtest.ts
 create mode 100644 internal/memtest/package.json
 create mode 100644 internal/memtest/src/index.ts
 create mode 100644 internal/memtest/src/loadtest-script.ts
 create mode 100644 internal/memtest/src/loadtest.ts

diff --git a/e2e/federation-example/federation-example.memtest.ts b/e2e/federation-example/federation-example.memtest.ts
new file mode 100644
index 000000000..000a9876f
--- /dev/null
+++ b/e2e/federation-example/federation-example.memtest.ts
@@ -0,0 +1,22 @@
+import { createExampleSetup, createTenv } from '@internal/e2e';
+import { loadtest } from '@internal/memtest';
+import { expect, it } from 'vitest';
+
+const { gateway } = createTenv(__dirname);
+const { supergraph, query } = createExampleSetup(__dirname);
+
+it('should not leak', async () => {
+  const gw = await gateway({
+    supergraph: await supergraph(),
+  });
+
+  using test = await loadtest({
+    cwd: __dirname,
+    server: gw,
+    query,
+  });
+
+  await test.waitForComplete;
+
+  expect(() => test.checkMemTrend()).not.toThrow();
+});
diff --git a/internal/e2e/src/tenv.ts b/internal/e2e/src/tenv.ts
index c572d4b2e..bb1c97ce4 100644
--- a/internal/e2e/src/tenv.ts
+++ b/internal/e2e/src/tenv.ts
@@ -14,7 +14,7 @@ import {
   fakePromise,
   registerAbortSignalListener,
 } from '@graphql-tools/utils';
-import { Proc, ProcOptions, spawn, waitForPort } from '@internal/proc';
+import { Proc, ProcOptions, Server, spawn, waitForPort } from '@internal/proc';
 import {
   boolEnv,
   createOpt,
@@ -82,11 +82,6 @@ yarn build && E2E_GATEWAY_RUNNER=bun-docker yarn workspace @graphql-hive/gateway
   return runner as ServeRunner;
 })();
 
-export interface Server extends Proc {
-  port: number;
-  protocol: string;
-}
-
 export interface GatewayOptions extends ProcOptions {
   port?: number;
   /**
@@ -552,6 +547,7 @@ export function createTenv(cwd: string): Tenv {
         ...proc,
         port,
         protocol,
+        url: `${protocol}://0.0.0.0:${port}`, // TODO: don't depend on localhost/0.0.0.0
         async execute({ headers, ...args }) {
           try {
             const res = await fetch(`${protocol}://0.0.0.0:${port}/graphql`, {
@@ -711,7 +707,13 @@ export function createTenv(cwd: string): Tenv {
         gatewayPort && createPortOpt(gatewayPort),
         ...args,
       );
-      const service: Service = { ...proc, name, port, protocol };
+      const service: Service = {
+        ...proc,
+        name,
+        port,
+        protocol,
+        url: `${protocol}://0.0.0.0:${port}`, // TODO: don't depend on localhost/0.0.0.0
+      };
       await Promise.race([
         waitForExit
           .then(() => {
@@ -874,6 +876,7 @@ export function createTenv(cwd: string): Tenv {
         name,
         port: hostPort,
         protocol,
+        url: `${protocol}://0.0.0.0:${hostPort}`, // TODO: don't depend on localhost/0.0.0.0
         additionalPorts,
         getStd() {
           // TODO: distinguish stdout and stderr
diff --git a/internal/memtest/package.json b/internal/memtest/package.json
new file mode 100644
index 000000000..9bce734f9
--- /dev/null
+++ b/internal/memtest/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "@internal/memtest",
+  "type": "module",
+  "private": true,
+  "main": "./src/index.ts",
+  "dependencies": {
+    "parse-duration": "^2.0.0",
+    "regression": "^2.0.1"
+  },
+  "devDependencies": {
+    "@types/k6": "^0.54.2",
+    "@types/regression": "^2"
+  }
+}
diff --git a/internal/memtest/src/index.ts b/internal/memtest/src/index.ts
new file mode 100644
index 000000000..43762c41b
--- /dev/null
+++ b/internal/memtest/src/index.ts
@@ -0,0 +1 @@
+export * from './loadtest';
diff --git a/internal/memtest/src/loadtest-script.ts b/internal/memtest/src/loadtest-script.ts
new file mode 100644
index 000000000..0d6fdb1f9
--- /dev/null
+++ b/internal/memtest/src/loadtest-script.ts
@@ -0,0 +1,32 @@
+import { check } from 'k6';
+import { test } from 'k6/execution';
+import http from 'k6/http';
+
+export default function () {
+  const url = __ENV['URL'];
+  if (!url) {
+    return test.abort('Environment variable "URL" not provided');
+  }
+
+  const query = __ENV['QUERY'];
+  if (!query) {
+    return test.abort('Environment variable "QUERY" not provided');
+  }
+
+  check(
+    http.post(
+      url,
+      { query },
+      {
+        headers: {
+          'content-type': 'application/json',
+        },
+      },
+    ),
+    {
+      'status is 200': (res) => res.status === 200,
+      'body contains data': (res) =>
+        !!res.body?.toString().includes('"data":{'),
+    },
+  );
+}
diff --git a/internal/memtest/src/loadtest.ts b/internal/memtest/src/loadtest.ts
new file mode 100644
index 000000000..2b0fc62d8
--- /dev/null
+++ b/internal/memtest/src/loadtest.ts
@@ -0,0 +1,130 @@
+import path from 'path';
+import { setTimeout } from 'timers/promises';
+import { ProcOptions, Server, spawn } from '@internal/proc';
+import parseDuration from 'parse-duration';
+import * as regression from 'regression';
+
+export interface LoadtestOptions extends ProcOptions {
+  cwd: string;
+  /** @default 100 */
+  vus?: number;
+  /** @default 30s */
+  duration?: string;
+  /**
+   * The memory increase threshold for the slope in the regression line of the memory snapshots.
+   * @default 10
+   */
+  memoryIncreaseTrendThresholdInMB?: number;
+  /** The GraphQL server on which the loadtest is running. */
+  server: Server;
+  /**
+   * The GraphQL query to execute for the loadtest.
+   */
+  query: string;
+}
+
+export async function loadtest(opts: LoadtestOptions) {
+  const {
+    cwd,
+    vus = 100,
+    duration = '30s',
+    memoryIncreaseTrendThresholdInMB = 10,
+    server,
+    query,
+    ...procOptions
+  } = opts;
+
+  const durationInMs = parseDuration(duration);
+  if (!durationInMs) {
+    throw new Error(`Cannot parse duration "${duration}" to milliseconds`);
+  }
+  if (durationInMs < 3_000) {
+    throw new Error(`Duration has to be at least 3s, got "${duration}"`);
+  }
+
+  const ctrl = new AbortController();
+  const signal = AbortSignal.any([
+    ctrl.signal,
+    AbortSignal.timeout(
+      durationInMs +
+        // allow 1s for the k6 process to exit gracefully
+        1_000,
+    ),
+  ]);
+
+  const [, waitForExit] = await spawn(
+    {
+      cwd,
+      ...procOptions,
+      signal,
+    },
+    'k6',
+    'run',
+    `--vus=${vus}`,
+    `--duration=${duration}`,
+    `--env=URL=${server.url + '/graphql'}`,
+    `--env=QUERY=${query}`,
+    path.join(__dirname, 'loadtest-script.ts'),
+  );
+
+  const memInMbSnapshots: number[] = [];
+
+  // we dont use a `setInterval` because the proc.getStats is async and we want stats ordered by time
+  (async () => {
+    // abort as soon as the loadtest exits breaking the mem snapshot loop
+    waitForExit.finally(() => ctrl.abort());
+
+    while (!signal.aborted) {
+      await setTimeout(1_000); // get memory snapshot every second
+      try {
+        const { mem } = await server.getStats();
+        memInMbSnapshots.push(mem);
+      } catch (err) {
+        if (!signal.aborted) {
+          throw err;
+        }
+        return; // couldve been aborted after timeout or while waiting for stats
+      }
+    }
+  })();
+
+  return {
+    waitForComplete: waitForExit,
+    memInMbSnapshots,
+    checkMemTrend: () =>
+      checkMemTrend(memInMbSnapshots, memoryIncreaseTrendThresholdInMB),
+    [Symbol.dispose]() {
+      ctrl.abort();
+    },
+  };
+}
+
+/**
+ * Detects a memory increase trend in an array of memory snapshots over time using linear regression.
+ *
+ * @param snapshots - An array of memory snapshots in MB.
+ * @param threshold - The minimum slope to consider as a significant increase.
+ *
+ * @throws Error if there is an increase trend, with details about the slope.
+ */
+function checkMemTrend(snapshots: number[], threshold: number): void {
+  if (snapshots.length < 2) {
+    throw new Error('Not enough memory snapshots to determine trend');
+  }
+
+  const data: [x: number, y: number][] = snapshots.map((memInMB, timestamp) => [
+    timestamp,
+    memInMB,
+  ]);
+  const result = regression.linear(data);
+  const slope = result.equation[0];
+  if (!slope) {
+    throw new Error('Regression slope is zero');
+  }
+
+  if (slope > threshold) {
+    throw new Error(
+      `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${threshold}MB)`,
+    );
+  }
+}
diff --git a/internal/proc/src/index.ts b/internal/proc/src/index.ts
index 3acd4a087..e22354bbf 100644
--- a/internal/proc/src/index.ts
+++ b/internal/proc/src/index.ts
@@ -18,6 +18,12 @@ export interface Proc extends AsyncDisposable {
   }>;
 }
 
+export interface Server extends Proc {
+  port: number;
+  protocol: string;
+  url: string;
+}
+
 export interface ProcOptions {
   /**
    * Pipe the logs from the spawned process to the current process, or to a file
diff --git a/package.json b/package.json
index 0891c5122..bc2bbe2e6 100644
--- a/package.json
+++ b/package.json
@@ -23,7 +23,8 @@
     "test": "vitest --project unit",
     "test:bun": "bun test --bail",
     "test:e2e": "vitest --project e2e",
-    "test:leaks": "cross-env \"LEAK_TEST=1\" jest --detectOpenHandles --detectLeaks"
+    "test:leaks": "cross-env \"LEAK_TEST=1\" jest --detectOpenHandles --detectLeaks",
+    "test:memtest": "vitest --project memtest"
   },
   "devDependencies": {
     "@babel/core": "7.26.7",
diff --git a/tsconfig.json b/tsconfig.json
index 012777265..09219e63a 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -14,6 +14,7 @@
       "@internal/testing": ["./internal/testing/src/index.ts"],
       "@internal/e2e": ["./internal/e2e/src/index.ts"],
       "@internal/proc": ["./internal/proc/src/index.ts"],
+      "@internal/memtest": ["./internal/memtest/src/index.ts"],
       "@internal/testing/to-be-similar-string": [
         "./internal/testing/src/to-be-similar-string.ts"
       ],
diff --git a/vitest.projects.ts b/vitest.projects.ts
index 3e7df6045..e894660f1 100644
--- a/vitest.projects.ts
+++ b/vitest.projects.ts
@@ -44,4 +44,13 @@ export default defineWorkspace([
       },
     },
   },
+  {
+    extends: './vitest.config.ts',
+    test: {
+      name: 'memtest',
+      include: ['**/*.memtest.ts'],
+      hookTimeout: testTimeout,
+      testTimeout: 5 * 60 * 1_000, // 5 minutes (loadtest runs for 3 minutes)
+    },
+  },
 ]);
diff --git a/yarn.lock b/yarn.lock
index 91d262dde..6732406f2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4644,6 +4644,17 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@internal/memtest@workspace:internal/memtest":
+  version: 0.0.0-use.local
+  resolution: "@internal/memtest@workspace:internal/memtest"
+  dependencies:
+    "@types/k6": "npm:^0.54.2"
+    "@types/regression": "npm:^2"
+    parse-duration: "npm:^2.0.0"
+    regression: "npm:^2.0.1"
+  languageName: unknown
+  linkType: soft
+
 "@internal/proc@workspace:internal/proc":
   version: 0.0.0-use.local
   resolution: "@internal/proc@workspace:internal/proc"
@@ -6540,6 +6551,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/k6@npm:^0.54.2":
+  version: 0.54.2
+  resolution: "@types/k6@npm:0.54.2"
+  checksum: 10c0/8d44dbd11ac53cf426abed3dd3bba0a35b355b3ad18b9d220ec5295259d6a4ee5bb97d0b2c38ff6896b085a1ad151678498df137c66aafae6b760b746bb84be8
+  languageName: node
+  linkType: hard
+
 "@types/keyv@npm:^3.1.4":
   version: 3.1.4
   resolution: "@types/keyv@npm:3.1.4"
@@ -6626,6 +6644,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/regression@npm:^2":
+  version: 2.0.6
+  resolution: "@types/regression@npm:2.0.6"
+  checksum: 10c0/c188d7cb023c9c77436b3522eb2ffd0f92e92c36230e591e13b0c45dc69c77d820498ecd6a2b4208ea3cb81df5771e24101732c6b69a5e30d7897406c5f92c18
+  languageName: node
+  linkType: hard
+
 "@types/resolve@npm:1.20.2":
   version: 1.20.2
   resolution: "@types/resolve@npm:1.20.2"
@@ -15142,6 +15167,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"regression@npm:^2.0.1":
+  version: 2.0.1
+  resolution: "regression@npm:2.0.1"
+  checksum: 10c0/ad3ca209409ed6e3ce1da64ff9779f616c1ef5cfdfc0785681aa670a046d674c8dcdaeac2a7c41f9cb3c22374051246d675e79c2ba3885deb6c9f9aadc756cd8
+  languageName: node
+  linkType: hard
+
 "relateurl@npm:^0.2.7":
   version: 0.2.7
   resolution: "relateurl@npm:0.2.7"

From 45b2439fc783bcfea492969e1cd88ee2ab3631c8 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 19:37:22 +0100
Subject: [PATCH 02/19] no url and slope length

---
 internal/e2e/src/tenv.ts         | 3 ---
 internal/memtest/src/loadtest.ts | 6 +++---
 internal/proc/src/index.ts       | 1 -
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/internal/e2e/src/tenv.ts b/internal/e2e/src/tenv.ts
index bb1c97ce4..5730d6da3 100644
--- a/internal/e2e/src/tenv.ts
+++ b/internal/e2e/src/tenv.ts
@@ -547,7 +547,6 @@ export function createTenv(cwd: string): Tenv {
         ...proc,
         port,
         protocol,
-        url: `${protocol}://0.0.0.0:${port}`, // TODO: don't depend on localhost/0.0.0.0
         async execute({ headers, ...args }) {
           try {
             const res = await fetch(`${protocol}://0.0.0.0:${port}/graphql`, {
@@ -712,7 +711,6 @@ export function createTenv(cwd: string): Tenv {
         name,
         port,
         protocol,
-        url: `${protocol}://0.0.0.0:${port}`, // TODO: don't depend on localhost/0.0.0.0
       };
       await Promise.race([
         waitForExit
@@ -876,7 +874,6 @@ export function createTenv(cwd: string): Tenv {
         name,
         port: hostPort,
         protocol,
-        url: `${protocol}://0.0.0.0:${hostPort}`, // TODO: don't depend on localhost/0.0.0.0
         additionalPorts,
         getStd() {
           // TODO: distinguish stdout and stderr
diff --git a/internal/memtest/src/loadtest.ts b/internal/memtest/src/loadtest.ts
index 2b0fc62d8..ded3a1e73 100644
--- a/internal/memtest/src/loadtest.ts
+++ b/internal/memtest/src/loadtest.ts
@@ -62,7 +62,7 @@ export async function loadtest(opts: LoadtestOptions) {
     'run',
     `--vus=${vus}`,
     `--duration=${duration}`,
-    `--env=URL=${server.url + '/graphql'}`,
+    `--env=URL=${server.protocol}://localhost:${server.port}/graphql`,
     `--env=QUERY=${query}`,
     path.join(__dirname, 'loadtest-script.ts'),
   );
@@ -102,7 +102,7 @@ export async function loadtest(opts: LoadtestOptions) {
 /**
  * Detects a memory increase trend in an array of memory snapshots over time using linear regression.
  *
- * @param snapshots - An array of memory snapshots in MB.
+ * @param snapshots - An array of memory snapshots in MB distanced by 1 second.
  * @param threshold - The minimum slope to consider as a significant increase.
  *
  * @throws Error if there is an increase trend, with details about the slope.
@@ -124,7 +124,7 @@ function checkMemTrend(snapshots: number[], threshold: number): void {
 
   if (slope > threshold) {
     throw new Error(
-      `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${threshold}MB)`,
+      `Memory increase trend detected with slope of ${slope}MB over ${snapshots.length}s (exceding threshold of ${threshold}MB)`,
     );
   }
 }
diff --git a/internal/proc/src/index.ts b/internal/proc/src/index.ts
index e22354bbf..1420e4cd3 100644
--- a/internal/proc/src/index.ts
+++ b/internal/proc/src/index.ts
@@ -21,7 +21,6 @@ export interface Proc extends AsyncDisposable {
 export interface Server extends Proc {
   port: number;
   protocol: string;
-  url: string;
 }
 
 export interface ProcOptions {

From b21a81c9cf0b11af61ba14a3bff205d848e4c8de Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 19:39:41 +0100
Subject: [PATCH 03/19] fix loadtest script

---
 internal/memtest/src/loadtest-script.ts | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/internal/memtest/src/loadtest-script.ts b/internal/memtest/src/loadtest-script.ts
index 0d6fdb1f9..268742a93 100644
--- a/internal/memtest/src/loadtest-script.ts
+++ b/internal/memtest/src/loadtest-script.ts
@@ -13,20 +13,10 @@ export default function () {
     return test.abort('Environment variable "QUERY" not provided');
   }
 
-  check(
-    http.post(
-      url,
-      { query },
-      {
-        headers: {
-          'content-type': 'application/json',
-        },
-      },
-    ),
-    {
-      'status is 200': (res) => res.status === 200,
-      'body contains data': (res) =>
-        !!res.body?.toString().includes('"data":{'),
-    },
-  );
+  const res = http.post(url, { query });
+
+  check(res, {
+    'status is 200': (res) => res.status === 200,
+    'body contains data': (res) => !!res.body?.toString().includes('"data":{'),
+  });
 }

From 9f0a54d8423104dd54cbd7a6908f429d0423088f Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 19:45:10 +0100
Subject: [PATCH 04/19] simplify

---
 internal/memtest/src/loadtest.ts | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/internal/memtest/src/loadtest.ts b/internal/memtest/src/loadtest.ts
index ded3a1e73..9aaf9a04d 100644
--- a/internal/memtest/src/loadtest.ts
+++ b/internal/memtest/src/loadtest.ts
@@ -11,10 +11,10 @@ export interface LoadtestOptions extends ProcOptions {
   /** @default 30s */
   duration?: string;
   /**
-   * The memory increase threshold for the slope in the regression line of the memory snapshots.
+   * Linear regression line slope threshold of the memory snapshots.
    * @default 10
    */
-  memoryIncreaseTrendThresholdInMB?: number;
+  memoryThresholdInMB?: number;
   /** The GraphQL server on which the loadtest is running. */
   server: Server;
   /**
@@ -28,7 +28,7 @@ export async function loadtest(opts: LoadtestOptions) {
     cwd,
     vus = 100,
     duration = '30s',
-    memoryIncreaseTrendThresholdInMB = 10,
+    memoryThresholdInMB = 10,
     server,
     query,
     ...procOptions
@@ -91,8 +91,7 @@ export async function loadtest(opts: LoadtestOptions) {
   return {
     waitForComplete: waitForExit,
     memInMbSnapshots,
-    checkMemTrend: () =>
-      checkMemTrend(memInMbSnapshots, memoryIncreaseTrendThresholdInMB),
+    checkMemTrend: () => checkMemTrend(memInMbSnapshots, memoryThresholdInMB),
     [Symbol.dispose]() {
       ctrl.abort();
     },

From 526bf7613e9ef7a63ff236cd5e5118dcd8fa4e32 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 19:54:38 +0100
Subject: [PATCH 05/19] internal perf

---
 e2e/federation-example/federation-example.memtest.ts | 2 +-
 internal/{memtest => perf}/package.json              | 2 +-
 internal/{memtest => perf}/src/index.ts              | 0
 internal/{memtest => perf}/src/loadtest-script.ts    | 0
 internal/{memtest => perf}/src/loadtest.ts           | 0
 tsconfig.json                                        | 2 +-
 yarn.lock                                            | 4 ++--
 7 files changed, 5 insertions(+), 5 deletions(-)
 rename internal/{memtest => perf}/package.json (88%)
 rename internal/{memtest => perf}/src/index.ts (100%)
 rename internal/{memtest => perf}/src/loadtest-script.ts (100%)
 rename internal/{memtest => perf}/src/loadtest.ts (100%)

diff --git a/e2e/federation-example/federation-example.memtest.ts b/e2e/federation-example/federation-example.memtest.ts
index 000a9876f..aefb23585 100644
--- a/e2e/federation-example/federation-example.memtest.ts
+++ b/e2e/federation-example/federation-example.memtest.ts
@@ -1,5 +1,5 @@
 import { createExampleSetup, createTenv } from '@internal/e2e';
-import { loadtest } from '@internal/memtest';
+import { loadtest } from '@internal/perf';
 import { expect, it } from 'vitest';
 
 const { gateway } = createTenv(__dirname);
diff --git a/internal/memtest/package.json b/internal/perf/package.json
similarity index 88%
rename from internal/memtest/package.json
rename to internal/perf/package.json
index 9bce734f9..adb9a6f33 100644
--- a/internal/memtest/package.json
+++ b/internal/perf/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "@internal/memtest",
+  "name": "@internal/perf",
   "type": "module",
   "private": true,
   "main": "./src/index.ts",
diff --git a/internal/memtest/src/index.ts b/internal/perf/src/index.ts
similarity index 100%
rename from internal/memtest/src/index.ts
rename to internal/perf/src/index.ts
diff --git a/internal/memtest/src/loadtest-script.ts b/internal/perf/src/loadtest-script.ts
similarity index 100%
rename from internal/memtest/src/loadtest-script.ts
rename to internal/perf/src/loadtest-script.ts
diff --git a/internal/memtest/src/loadtest.ts b/internal/perf/src/loadtest.ts
similarity index 100%
rename from internal/memtest/src/loadtest.ts
rename to internal/perf/src/loadtest.ts
diff --git a/tsconfig.json b/tsconfig.json
index 09219e63a..cf23cba27 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -14,7 +14,7 @@
       "@internal/testing": ["./internal/testing/src/index.ts"],
       "@internal/e2e": ["./internal/e2e/src/index.ts"],
       "@internal/proc": ["./internal/proc/src/index.ts"],
-      "@internal/memtest": ["./internal/memtest/src/index.ts"],
+      "@internal/perf": ["./internal/loadtest/src/index.ts"],
       "@internal/testing/to-be-similar-string": [
         "./internal/testing/src/to-be-similar-string.ts"
       ],
diff --git a/yarn.lock b/yarn.lock
index 6732406f2..b518789ac 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4644,9 +4644,9 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@internal/memtest@workspace:internal/memtest":
+"@internal/perf@workspace:internal/perf":
   version: 0.0.0-use.local
-  resolution: "@internal/memtest@workspace:internal/memtest"
+  resolution: "@internal/perf@workspace:internal/perf"
   dependencies:
     "@types/k6": "npm:^0.54.2"
     "@types/regression": "npm:^2"

From 82a93aec1ba350512d7fbedf366a512b7507474b Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:12:04 +0100
Subject: [PATCH 06/19] memtest that actually runs the test

---
 .../federation-example.memtest.ts             | 31 ++++----
 internal/perf/src/loadtest.ts                 | 66 ++++------------
 internal/perf/src/memtest.ts                  | 76 +++++++++++++++++++
 tsconfig.json                                 |  3 +-
 vitest.projects.ts                            |  2 +-
 5 files changed, 108 insertions(+), 70 deletions(-)
 create mode 100644 internal/perf/src/memtest.ts

diff --git a/e2e/federation-example/federation-example.memtest.ts b/e2e/federation-example/federation-example.memtest.ts
index aefb23585..70a3a70aa 100644
--- a/e2e/federation-example/federation-example.memtest.ts
+++ b/e2e/federation-example/federation-example.memtest.ts
@@ -1,22 +1,19 @@
 import { createExampleSetup, createTenv } from '@internal/e2e';
-import { loadtest } from '@internal/perf';
-import { expect, it } from 'vitest';
+import { memtest } from '@internal/perf/memtest';
 
-const { gateway } = createTenv(__dirname);
-const { supergraph, query } = createExampleSetup(__dirname);
+const cwd = __dirname;
 
-it('should not leak', async () => {
-  const gw = await gateway({
-    supergraph: await supergraph(),
-  });
+const { gateway } = createTenv(cwd);
+const { supergraph, query } = createExampleSetup(cwd);
 
-  using test = await loadtest({
-    cwd: __dirname,
-    server: gw,
+memtest(
+  {
+    cwd,
     query,
-  });
-
-  await test.waitForComplete;
-
-  expect(() => test.checkMemTrend()).not.toThrow();
-});
+    pipeLogs: 'loadtest.out',
+  },
+  async () =>
+    gateway({
+      supergraph: await supergraph(),
+    }),
+);
diff --git a/internal/perf/src/loadtest.ts b/internal/perf/src/loadtest.ts
index 9aaf9a04d..8deae42ac 100644
--- a/internal/perf/src/loadtest.ts
+++ b/internal/perf/src/loadtest.ts
@@ -1,20 +1,19 @@
 import path from 'path';
 import { setTimeout } from 'timers/promises';
 import { ProcOptions, Server, spawn } from '@internal/proc';
-import parseDuration from 'parse-duration';
-import * as regression from 'regression';
 
 export interface LoadtestOptions extends ProcOptions {
   cwd: string;
   /** @default 100 */
   vus?: number;
-  /** @default 30s */
-  duration?: string;
+  /** Duration of the loadtest in milliseconds. */
+  duration: number;
   /**
-   * Linear regression line slope threshold of the memory snapshots.
-   * @default 10
+   * The snapshotting window of the GraphQL server memory in milliseconds.
+   *
+   * @default 1_000
    */
-  memoryThresholdInMB?: number;
+  memorySnapshotWindow?: number;
   /** The GraphQL server on which the loadtest is running. */
   server: Server;
   /**
@@ -27,18 +26,14 @@ export async function loadtest(opts: LoadtestOptions) {
   const {
     cwd,
     vus = 100,
-    duration = '30s',
-    memoryThresholdInMB = 10,
+    duration,
+    memorySnapshotWindow = 1_000,
     server,
     query,
     ...procOptions
   } = opts;
 
-  const durationInMs = parseDuration(duration);
-  if (!durationInMs) {
-    throw new Error(`Cannot parse duration "${duration}" to milliseconds`);
-  }
-  if (durationInMs < 3_000) {
+  if (duration < 3_000) {
     throw new Error(`Duration has to be at least 3s, got "${duration}"`);
   }
 
@@ -46,7 +41,7 @@ export async function loadtest(opts: LoadtestOptions) {
   const signal = AbortSignal.any([
     ctrl.signal,
     AbortSignal.timeout(
-      durationInMs +
+      duration +
         // allow 1s for the k6 process to exit gracefully
         1_000,
     ),
@@ -61,13 +56,13 @@ export async function loadtest(opts: LoadtestOptions) {
     'k6',
     'run',
     `--vus=${vus}`,
-    `--duration=${duration}`,
+    `--duration=${duration}ms`,
     `--env=URL=${server.protocol}://localhost:${server.port}/graphql`,
     `--env=QUERY=${query}`,
     path.join(__dirname, 'loadtest-script.ts'),
   );
 
-  const memInMbSnapshots: number[] = [];
+  const memoryInMBSnapshots: number[] = [];
 
   // we dont use a `setInterval` because the proc.getStats is async and we want stats ordered by time
   (async () => {
@@ -75,10 +70,10 @@ export async function loadtest(opts: LoadtestOptions) {
     waitForExit.finally(() => ctrl.abort());
 
     while (!signal.aborted) {
-      await setTimeout(1_000); // get memory snapshot every second
+      await setTimeout(memorySnapshotWindow);
       try {
         const { mem } = await server.getStats();
-        memInMbSnapshots.push(mem);
+        memoryInMBSnapshots.push(mem);
       } catch (err) {
         if (!signal.aborted) {
           throw err;
@@ -90,40 +85,9 @@ export async function loadtest(opts: LoadtestOptions) {
 
   return {
     waitForComplete: waitForExit,
-    memInMbSnapshots,
-    checkMemTrend: () => checkMemTrend(memInMbSnapshots, memoryThresholdInMB),
+    memoryInMBSnapshots,
     [Symbol.dispose]() {
       ctrl.abort();
     },
   };
 }
-
-/**
- * Detects a memory increase trend in an array of memory snapshots over time using linear regression.
- *
- * @param snapshots - An array of memory snapshots in MB distanced by 1 second.
- * @param threshold - The minimum slope to consider as a significant increase.
- *
- * @throws Error if there is an increase trend, with details about the slope.
- */
-function checkMemTrend(snapshots: number[], threshold: number): void {
-  if (snapshots.length < 2) {
-    throw new Error('Not enough memory snapshots to determine trend');
-  }
-
-  const data: [x: number, y: number][] = snapshots.map((memInMB, timestamp) => [
-    timestamp,
-    memInMB,
-  ]);
-  const result = regression.linear(data);
-  const slope = result.equation[0];
-  if (!slope) {
-    throw new Error('Regression slope is zero');
-  }
-
-  if (slope > threshold) {
-    throw new Error(
-      `Memory increase trend detected with slope of ${slope}MB over ${snapshots.length}s (exceding threshold of ${threshold}MB)`,
-    );
-  }
-}
diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
new file mode 100644
index 000000000..b6b5fbcde
--- /dev/null
+++ b/internal/perf/src/memtest.ts
@@ -0,0 +1,76 @@
+import { Server } from '@internal/proc';
+import regression from 'regression';
+import { it } from 'vitest';
+import { loadtest, LoadtestOptions } from './loadtest';
+
+export interface MemtestOptions
+  extends Omit<LoadtestOptions, 'duration' | 'server'> {
+  /**
+   * Duration of the loadtest in milliseconds.
+   *
+   * @default 30_000
+   */
+  duration?: number;
+  /**
+   * Linear regression line slope threshold of the memory snapshots.
+   * If the slope is greater than this value, the test will fail.
+   *
+   * @default 10
+   */
+  memoryThresholdInMB?: number;
+}
+
+export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
+  const { memoryThresholdInMB = 10, duration = 30_000, ...loadtestOpts } = opts;
+  it(
+    'should not have a memory increase trend',
+    async ({ expect }) => {
+      const server = await setup();
+
+      using test = await loadtest({
+        ...loadtestOpts,
+        duration,
+        server,
+      });
+
+      await test.waitForComplete;
+
+      expect(() =>
+        checkMemTrend(test.memoryInMBSnapshots, memoryThresholdInMB),
+      ).not.toThrow();
+    },
+    {
+      timeout: duration + 5_000, // allow 5s for the test to finish
+    },
+  );
+}
+
+/**
+ * Detects a memory increase trend in an array of memory snapshots over time using linear regression.
+ *
+ * @param snapshots - An array of memory snapshots in MB.
+ * @param threshold - The minimum slope to consider as a significant increase.
+ *
+ * @throws Error if there is an increase trend, with details about the slope.
+ */
+function checkMemTrend(snapshots: number[], threshold: number): void {
+  if (snapshots.length < 2) {
+    throw new Error('Not enough memory snapshots to determine trend');
+  }
+
+  const data: [x: number, y: number][] = snapshots.map((memInMB, timestamp) => [
+    timestamp,
+    memInMB,
+  ]);
+  const result = regression.linear(data);
+  const slope = result.equation[0];
+  if (!slope) {
+    throw new Error('Regression slope is zero');
+  }
+
+  if (slope > threshold) {
+    throw new Error(
+      `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${threshold}MB)`,
+    );
+  }
+}
diff --git a/tsconfig.json b/tsconfig.json
index cf23cba27..3c7e46cbc 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -14,7 +14,8 @@
       "@internal/testing": ["./internal/testing/src/index.ts"],
       "@internal/e2e": ["./internal/e2e/src/index.ts"],
       "@internal/proc": ["./internal/proc/src/index.ts"],
-      "@internal/perf": ["./internal/loadtest/src/index.ts"],
+      "@internal/perf": ["./internal/perf/src/index.ts"],
+      "@internal/perf/memtest": ["./internal/perf/src/memtest.ts"],
       "@internal/testing/to-be-similar-string": [
         "./internal/testing/src/to-be-similar-string.ts"
       ],
diff --git a/vitest.projects.ts b/vitest.projects.ts
index e894660f1..b7d8df15a 100644
--- a/vitest.projects.ts
+++ b/vitest.projects.ts
@@ -50,7 +50,7 @@ export default defineWorkspace([
       name: 'memtest',
       include: ['**/*.memtest.ts'],
       hookTimeout: testTimeout,
-      testTimeout: 5 * 60 * 1_000, // 5 minutes (loadtest runs for 3 minutes)
+      testTimeout,
     },
   },
 ]);

From 44ea292ba124989e808e2c7f1c4dc3afda2942a9 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:16:32 +0100
Subject: [PATCH 07/19] nicer message

---
 internal/perf/src/memtest.ts | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
index b6b5fbcde..3936b2f24 100644
--- a/internal/perf/src/memtest.ts
+++ b/internal/perf/src/memtest.ts
@@ -35,9 +35,12 @@ export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
 
       await test.waitForComplete;
 
-      expect(() =>
-        checkMemTrend(test.memoryInMBSnapshots, memoryThresholdInMB),
-      ).not.toThrow();
+      const slope = calculateRegressionSlope(test.memoryInMBSnapshots);
+
+      expect(
+        slope,
+        `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${memoryThresholdInMB}MB)`,
+      ).toBeLessThan(memoryThresholdInMB);
     },
     {
       timeout: duration + 5_000, // allow 5s for the test to finish
@@ -49,13 +52,12 @@ export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
  * Detects a memory increase trend in an array of memory snapshots over time using linear regression.
  *
  * @param snapshots - An array of memory snapshots in MB.
- * @param threshold - The minimum slope to consider as a significant increase.
  *
- * @throws Error if there is an increase trend, with details about the slope.
+ * @returns The slope of the linear regression line.
  */
-function checkMemTrend(snapshots: number[], threshold: number): void {
+function calculateRegressionSlope(snapshots: number[]) {
   if (snapshots.length < 2) {
-    throw new Error('Not enough memory snapshots to determine trend');
+    throw new Error('Not enough snapshots to determine trend');
   }
 
   const data: [x: number, y: number][] = snapshots.map((memInMB, timestamp) => [
@@ -68,9 +70,5 @@ function checkMemTrend(snapshots: number[], threshold: number): void {
     throw new Error('Regression slope is zero');
   }
 
-  if (slope > threshold) {
-    throw new Error(
-      `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${threshold}MB)`,
-    );
-  }
+  return slope;
 }

From 546c66f14d9642e8e05327845661bdff8c2d8cf4 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:20:59 +0100
Subject: [PATCH 08/19] cleanup

---
 internal/perf/src/loadtest.ts | 18 ++++++++----------
 internal/perf/src/memtest.ts  | 12 +++++-------
 2 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/internal/perf/src/loadtest.ts b/internal/perf/src/loadtest.ts
index 8deae42ac..88c658a19 100644
--- a/internal/perf/src/loadtest.ts
+++ b/internal/perf/src/loadtest.ts
@@ -61,14 +61,16 @@ export async function loadtest(opts: LoadtestOptions) {
     `--env=QUERY=${query}`,
     path.join(__dirname, 'loadtest-script.ts'),
   );
+  using _ = {
+    [Symbol.dispose]() {
+      ctrl.abort();
+    },
+  };
 
   const memoryInMBSnapshots: number[] = [];
 
   // we dont use a `setInterval` because the proc.getStats is async and we want stats ordered by time
   (async () => {
-    // abort as soon as the loadtest exits breaking the mem snapshot loop
-    waitForExit.finally(() => ctrl.abort());
-
     while (!signal.aborted) {
       await setTimeout(memorySnapshotWindow);
       try {
@@ -83,11 +85,7 @@ export async function loadtest(opts: LoadtestOptions) {
     }
   })();
 
-  return {
-    waitForComplete: waitForExit,
-    memoryInMBSnapshots,
-    [Symbol.dispose]() {
-      ctrl.abort();
-    },
-  };
+  await waitForExit;
+
+  return { memoryInMBSnapshots };
 }
diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
index 3936b2f24..7b0d82867 100644
--- a/internal/perf/src/memtest.ts
+++ b/internal/perf/src/memtest.ts
@@ -24,27 +24,25 @@ export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
   const { memoryThresholdInMB = 10, duration = 30_000, ...loadtestOpts } = opts;
   it(
     'should not have a memory increase trend',
+    {
+      timeout: duration + 5_000, // allow 5s for the test to finish
+    },
     async ({ expect }) => {
       const server = await setup();
 
-      using test = await loadtest({
+      const { memoryInMBSnapshots } = await loadtest({
         ...loadtestOpts,
         duration,
         server,
       });
 
-      await test.waitForComplete;
-
-      const slope = calculateRegressionSlope(test.memoryInMBSnapshots);
+      const slope = calculateRegressionSlope(memoryInMBSnapshots);
 
       expect(
         slope,
         `Memory increase trend detected with slope of ${slope}MB (exceding threshold of ${memoryThresholdInMB}MB)`,
       ).toBeLessThan(memoryThresholdInMB);
     },
-    {
-      timeout: duration + 5_000, // allow 5s for the test to finish
-    },
   );
 }
 

From 1f26ff3ccc310d27d6364633e74c9bd7f69b5820 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:32:45 +0100
Subject: [PATCH 09/19] workflow

---
 .github/workflows/memtest.yml | 32 ++++++++++++++++++++++++++++++++
 package.json                  |  2 +-
 2 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/memtest.yml

diff --git a/.github/workflows/memtest.yml b/.github/workflows/memtest.yml
new file mode 100644
index 000000000..d271f869e
--- /dev/null
+++ b/.github/workflows/memtest.yml
@@ -0,0 +1,32 @@
+name: Memtest
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  NODE_NO_WARNINGS: 1
+
+jobs:
+  memtest:
+    strategy:
+      matrix:
+        e2e_runner: [node, bun]
+    name: ${{matrix.e2e_runner}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up env
+        uses: the-guild-org/shared-config/setup@v1
+        with:
+          node-version-file: .node-version
+      - name: Test
+        run: yarn test:mem
+        env:
+          E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}
diff --git a/package.json b/package.json
index bc2bbe2e6..74966c49c 100644
--- a/package.json
+++ b/package.json
@@ -24,7 +24,7 @@
     "test:bun": "bun test --bail",
     "test:e2e": "vitest --project e2e",
     "test:leaks": "cross-env \"LEAK_TEST=1\" jest --detectOpenHandles --detectLeaks",
-    "test:memtest": "vitest --project memtest"
+    "test:mem": "vitest --project memtest"
   },
   "devDependencies": {
     "@babel/core": "7.26.7",

From b4813905e2954349802358ab722e5f1640fb0322 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:35:52 +0100
Subject: [PATCH 10/19] no pipe logs

---
 e2e/federation-example/federation-example.memtest.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/e2e/federation-example/federation-example.memtest.ts b/e2e/federation-example/federation-example.memtest.ts
index 70a3a70aa..1c546eb14 100644
--- a/e2e/federation-example/federation-example.memtest.ts
+++ b/e2e/federation-example/federation-example.memtest.ts
@@ -10,7 +10,6 @@ memtest(
   {
     cwd,
     query,
-    pipeLogs: 'loadtest.out',
   },
   async () =>
     gateway({

From 304dbacc4c3477ee8aa1f76177bada929a29194b Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:46:50 +0100
Subject: [PATCH 11/19] install k6

---
 .github/workflows/memtest.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/memtest.yml b/.github/workflows/memtest.yml
index d271f869e..7d2c2df4b 100644
--- a/.github/workflows/memtest.yml
+++ b/.github/workflows/memtest.yml
@@ -11,6 +11,7 @@ concurrency:
 
 env:
   NODE_NO_WARNINGS: 1
+  K6_VERSION: v0.56.0
 
 jobs:
   memtest:
@@ -22,6 +23,12 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install k6
+        run: |
+          mkdir -p "$HOME/.local/bin"
+          cd "$HOME/.local/bin"
+          curl https://github.com/grafana/k6/releases/download/${{ env.K6_VERSION }}/k6-${{ env.K6_VERSION }}-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+          echo "$PWD" >> $GITHUB_PATH
       - name: Set up env
         uses: the-guild-org/shared-config/setup@v1
         with:

From dd0451517117aef60d64c5767c95ea51f2435b98 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:47:35 +0100
Subject: [PATCH 12/19] skip memtests from examples

---
 internal/examples/src/convert.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/examples/src/convert.ts b/internal/examples/src/convert.ts
index 2458ec11e..7f973b6ea 100644
--- a/internal/examples/src/convert.ts
+++ b/internal/examples/src/convert.ts
@@ -180,6 +180,8 @@ export async function convertE2EToExample(config: ConvertE2EToExampleConfig) {
       !path.basename(extraDirOrFile).includes('.e2e.') &&
       // not a bench
       !path.basename(extraDirOrFile).includes('.bench.') &&
+      // not a memtest
+      !path.basename(extraDirOrFile).includes('.memtest.') &&
       // not a dockerile
       !path.basename(extraDirOrFile).includes('Dockerfile') &&
       // not test snapshots

From 4ac2cb6fad16bb48b0fa10315d50deed05f1a103 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:49:26 +0100
Subject: [PATCH 13/19] simplify bench.yml

---
 .github/workflows/bench.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 00164e779..68e7df879 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -11,7 +11,6 @@ concurrency:
 
 env:
   NODE_NO_WARNINGS: 1
-  CI: true
 
 jobs:
   bench:
@@ -22,7 +21,7 @@ jobs:
           - 10
           - 100
           - 1000
-    name: Benchmark / ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
+    name: ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -36,4 +35,3 @@ jobs:
         env:
           PRODUCTS_SIZE: ${{matrix.products_size}}
           E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}
-          CI: true

From 8874c954a73e5750ac246b4fc06e5b8069a78fba Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 20:51:12 +0100
Subject: [PATCH 14/19] loadtest duration minute

---
 internal/perf/src/memtest.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
index 7b0d82867..fe5d60c1b 100644
--- a/internal/perf/src/memtest.ts
+++ b/internal/perf/src/memtest.ts
@@ -8,7 +8,7 @@ export interface MemtestOptions
   /**
    * Duration of the loadtest in milliseconds.
    *
-   * @default 30_000
+   * @default 60_000
    */
   duration?: number;
   /**
@@ -21,7 +21,7 @@ export interface MemtestOptions
 }
 
 export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
-  const { memoryThresholdInMB = 10, duration = 30_000, ...loadtestOpts } = opts;
+  const { memoryThresholdInMB = 10, duration = 60_000, ...loadtestOpts } = opts;
   it(
     'should not have a memory increase trend',
     {

From 77ce4d43f875fbc15a8293384c5d6381b7269b9c Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 21:01:14 +0100
Subject: [PATCH 15/19] allow more time for test teardown

---
 internal/perf/src/memtest.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
index fe5d60c1b..743638b2b 100644
--- a/internal/perf/src/memtest.ts
+++ b/internal/perf/src/memtest.ts
@@ -25,7 +25,7 @@ export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
   it(
     'should not have a memory increase trend',
     {
-      timeout: duration + 5_000, // allow 5s for the test to finish
+      timeout: duration + 10_000, // allow 10s for the test teardown
     },
     async ({ expect }) => {
       const server = await setup();

From 636918cb865794539bf7743315d97fa69f4c2c37 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 21:08:16 +0100
Subject: [PATCH 16/19] stricter threshold

---
 internal/perf/src/memtest.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/perf/src/memtest.ts b/internal/perf/src/memtest.ts
index 743638b2b..54332ed59 100644
--- a/internal/perf/src/memtest.ts
+++ b/internal/perf/src/memtest.ts
@@ -15,13 +15,13 @@ export interface MemtestOptions
    * Linear regression line slope threshold of the memory snapshots.
    * If the slope is greater than this value, the test will fail.
    *
-   * @default 10
+   * @default 5
    */
   memoryThresholdInMB?: number;
 }
 
 export function memtest(opts: MemtestOptions, setup: () => Promise<Server>) {
-  const { memoryThresholdInMB = 10, duration = 60_000, ...loadtestOpts } = opts;
+  const { memoryThresholdInMB = 5, duration = 60_000, ...loadtestOpts } = opts;
   it(
     'should not have a memory increase trend',
     {

From 2275bafba9f6323fda98bb71ffb86623a3b45550 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 21:16:48 +0100
Subject: [PATCH 17/19] stop loadtest if server exits

---
 internal/perf/src/loadtest.ts | 2 +-
 internal/proc/src/index.ts    | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/perf/src/loadtest.ts b/internal/perf/src/loadtest.ts
index 88c658a19..954345c49 100644
--- a/internal/perf/src/loadtest.ts
+++ b/internal/perf/src/loadtest.ts
@@ -85,7 +85,7 @@ export async function loadtest(opts: LoadtestOptions) {
     }
   })();
 
-  await waitForExit;
+  await Promise.race([waitForExit, server.waitForExit]);
 
   return { memoryInMBSnapshots };
 }
diff --git a/internal/proc/src/index.ts b/internal/proc/src/index.ts
index 1420e4cd3..d836ef6c4 100644
--- a/internal/proc/src/index.ts
+++ b/internal/proc/src/index.ts
@@ -9,6 +9,7 @@ import { fetch } from '@whatwg-node/fetch';
 import terminate from 'terminate/promise';
 
 export interface Proc extends AsyncDisposable {
+  waitForExit: Promise<void>;
   getStd(o: 'out' | 'err' | 'both'): string;
   getStats(): Promise<{
     // Total CPU utilization (of all cores) as a percentage.
@@ -84,6 +85,7 @@ export function spawn(
   let stderr = '';
   let stdboth = '';
   const proc: Proc = {
+    waitForExit,
     getStd(o) {
       switch (o) {
         case 'out':

From e8d0474ea43349ffde1c619b15cfb1629b0a6f14 Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 21:29:26 +0100
Subject: [PATCH 18/19] server exited before loadtest is fail

---
 internal/perf/src/loadtest.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/internal/perf/src/loadtest.ts b/internal/perf/src/loadtest.ts
index 954345c49..3af9255f6 100644
--- a/internal/perf/src/loadtest.ts
+++ b/internal/perf/src/loadtest.ts
@@ -1,6 +1,7 @@
 import path from 'path';
 import { setTimeout } from 'timers/promises';
 import { ProcOptions, Server, spawn } from '@internal/proc';
+import { trimError } from '@internal/testing';
 
 export interface LoadtestOptions extends ProcOptions {
   cwd: string;
@@ -85,7 +86,14 @@ export async function loadtest(opts: LoadtestOptions) {
     }
   })();
 
-  await Promise.race([waitForExit, server.waitForExit]);
+  await Promise.race([
+    waitForExit,
+    server.waitForExit.then(() => {
+      throw new Error(
+        `Server exited before the loadtest finished\n${trimError(server.getStd('both'))}`,
+      );
+    }),
+  ]);
 
   return { memoryInMBSnapshots };
 }

From b491f0f396914cce8857ff2a3cb75ffb53758e6f Mon Sep 17 00:00:00 2001
From: Denis Badurina <denis@denelop.com>
Date: Thu, 6 Feb 2025 21:35:55 +0100
Subject: [PATCH 19/19] container wait for exit

---
 internal/e2e/src/tenv.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/e2e/src/tenv.ts b/internal/e2e/src/tenv.ts
index 5730d6da3..0edf733d8 100644
--- a/internal/e2e/src/tenv.ts
+++ b/internal/e2e/src/tenv.ts
@@ -870,6 +870,7 @@ export function createTenv(cwd: string): Tenv {
       await ctr.start();
 
       const container: Container = {
+        waitForExit: ctr.wait(),
         containerName,
         name,
         port: hostPort,