feat: Exit with 1 if a test failed (#58)

Author: Jason3S

examples/package.json

@@ -4,7 +4,6 @@
   "description": "Example perf tests.",
   "type": "module",
   "private": true,
-  "packageManager": "pnpm@8.15.7",
   "engines": {
     "node": ">=18"
   },

examples/src/fail.perf.mts

@@ -0,0 +1,19 @@
+import { suite } from 'perf-insight';
+
+// Use 2 seconds as the default timeout for tests in the suite.
+// The `--timeout` option can override this value.
+const defaultTimeout = 100;
+
+// ts-check
+suite('fail', 'Example with tests that fail or throw exceptions.', async (test) => {
+    test('ok', () => {
+        let a = '';
+        for (let i = 0; i < 1000; ++i) {
+            a = a + 'a';
+        }
+    });
+
+    test('fail', () => {
+        throw new Error('This test failed.');
+    });
+}).setTimeout(defaultTimeout); // set the default timeout for this suite.

package.json

@@ -49,7 +49,7 @@
     "@eslint/js": "^9.7.0",
     "@tsconfig/node20": "^20.1.4",
     "@types/node": "^20.14.10",
-    "@vitest/coverage-v8": "^1.6.0",
+    "@vitest/coverage-v8": "^2.0.2",
     "cspell": "^8.10.4",
     "cspell-trie-lib": "^8.10.4",
     "eslint": "^9.7.0",
@@ -68,6 +68,6 @@
     "typescript": "^5.5.3",
     "typescript-eslint": "^7.16.0",
     "vite": "^5.3.3",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   }
 }

packages/perf-insight/package.json

@@ -3,9 +3,8 @@
   "version": "1.1.1",
   "description": "Performance benchmarking tool for NodeJS.",
   "type": "module",
-  "packageManager": "pnpm@8.15.7",
   "engines": {
-    "node": ">=18"
+    "node": ">=18.18"
   },
   "bin": {
     "insight": "./bin.mjs",

packages/perf-insight/src/app.mts

@@ -15,6 +15,7 @@ interface AppOptions {
     suite?: string[];
     test?: string[];
     register?: string[];
+    failFast?: boolean;
 }
 
 const urlRunnerCli = new URL('./runBenchmarkCli.mjs', import.meta.url).toString();
@@ -34,6 +35,7 @@ export async function app(program = defaultCommand): Promise<Command> {
         .option('-t, --timeout <timeout>', 'Override the timeout for each test suite.', (v) => Number(v))
         .option('-s, --suite <suite...>', 'Run only matching suites.', appendValue)
         .option('-T, --test <test...>', 'Run only matching test found in suites', appendValue)
+        .option('--fail-fast', 'Stop on first failure.', false)
         .option('--repeat <count>', 'Repeat the tests.', (v) => Number(v), 1)
         .option('--register <loader>', 'Register a module loader. (e.g. ts-node/esm)', appendValue)
         .action(async (suiteNamesToRun: string[], options: AppOptions, command: Command) => {
@@ -62,7 +64,7 @@ export async function app(program = defaultCommand): Promise<Command> {
 
             await spawnRunners(files, options);
 
-            console.log(chalk.green('done.'));
+            process.exitCode ? console.log(chalk.red('failed.')) : console.log(chalk.green('done.'));
         });
 
     program.showHelpAfterError();
@@ -97,9 +99,16 @@ async function spawnRunners(files: string[], options: AppOptions): Promise<void>
     for (const file of files) {
         try {
             const code = await spawnRunner([file, ...cliOptions]);
-            code && console.error('Runner failed with "%s" code: %d', file, code);
+            if (code) {
+                // console.error('Runner failed with "%s" code: %d', file, code);
+                process.exitCode ??= code;
+                if (options.failFast) {
+                    break;
+                }
+            }
         } catch (e) {
             console.error('Failed to spawn runner.', e);
+            process.exitCode ??= 1;
         }
     }
 }

packages/perf-insight/src/perfSuite.mts

@@ -96,6 +96,7 @@ export interface RunnerResult {
     name: string;
     description: string | undefined;
     results: TestResult[];
+    hadFailures: boolean;
 }
 
 type TestMethod = () => void | Promise<void> | unknown | Promise<unknown>;
@@ -473,6 +474,7 @@ async function runTests(
         name,
         description,
         results,
+        hadFailures: results.some((r) => !!r.error),
     };
 }
 

packages/perf-insight/src/run.mts

@@ -11,22 +11,33 @@ export interface RunOptions {
     tests?: string[] | undefined;
 }
 
+export interface RunBenchmarkSuitesResult {
+    hadFailures: boolean;
+    numSuitesRun: number;
+}
+
 /**
  *
  * @param suiteNames
  * @param options
  */
-export async function runBenchmarkSuites(suiteToRun?: (string | PerfSuite)[], options?: RunOptions) {
+export async function runBenchmarkSuites(
+    suiteToRun?: (string | PerfSuite)[],
+    options?: RunOptions,
+): Promise<RunBenchmarkSuitesResult> {
     const suites = getActiveSuites();
 
     let numSuitesRun = 0;
     let showRepeatMsg = false;
+    let hadErrors = false;
 
     for (let repeat = options?.repeat || 1; repeat > 0; repeat--) {
         if (showRepeatMsg) {
             console.log(chalk.yellow(`Repeating tests: ${repeat} more time${repeat > 1 ? 's' : ''}.`));
         }
-        numSuitesRun = await runTestSuites(suites, suiteToRun || suites, options || {});
+        const r = await runTestSuites(suites, suiteToRun || suites, options || {});
+        numSuitesRun = r.numSuitesRun;
+        hadErrors ||= r.hadFailures;
         if (!numSuitesRun) break;
         showRepeatMsg = true;
     }
@@ -44,18 +55,32 @@ export async function runBenchmarkSuites(suiteToRun?: (string | PerfSuite)[], op
                 .map((line) => `  ${line}`)
                 .join('\n'),
         );
+
+        hadErrors = true;
     }
+
+    return { hadFailures: hadErrors, numSuitesRun };
+}
+
+interface Result {
+    hadFailures: boolean;
+}
+
+interface RunTestSuitesResults extends Result {
+    numSuitesRun: number;
 }
 
 async function runTestSuites(
     suites: PerfSuite[],
     suitesToRun: (string | PerfSuite)[],
     options: RunOptions,
-): Promise<number> {
+): Promise<RunTestSuitesResults> {
     const timeout = options.timeout || undefined;
     const suitesRun = new Set<PerfSuite>();
+    let hadFailures = false;
 
-    async function _runSuite(suites: PerfSuite[]) {
+    async function _runSuite(suites: PerfSuite[]): Promise<Result> {
+        let hadFailures = false;
         for (const suite of suites) {
             if (suitesRun.has(suite)) continue;
             if (!filterSuite(suite)) {
@@ -64,32 +89,36 @@ async function runTestSuites(
             }
             suitesRun.add(suite);
             console.log(chalk.green(`Running Perf Suite: ${suite.name}`));
-            await suite.setTimeout(timeout).runTests({ tests: options.tests });
+            const result = await suite.setTimeout(timeout).runTests({ tests: options.tests });
+            if (result.hadFailures) {
+                hadFailures = true;
+            }
         }
+
+        return { hadFailures: hadFailures };
     }
 
-    async function runSuite(name: string | PerfSuite) {
+    async function runSuite(name: string | PerfSuite): Promise<Result> {
         if (typeof name !== 'string') {
             return await _runSuite([name]);
         }
 
         if (name === 'all') {
-            await _runSuite(suites);
-            return;
+            return await _runSuite(suites);
         }
         const matching = suites.filter((suite) => suite.name.toLowerCase().startsWith(name.toLowerCase()));
         if (!matching.length) {
             console.log(chalk.red(`Unknown test method: ${name}`));
-            return;
+            return { hadFailures: true };
         }
-        await _runSuite(matching);
+        return await _runSuite(matching);
     }
 
     for (const name of suitesToRun) {
-        await runSuite(name);
+        hadFailures ||= (await runSuite(name)).hadFailures;
     }
 
-    return suitesRun.size;
+    return { hadFailures, numSuitesRun: suitesRun.size };
 
     function filterSuite(suite: PerfSuite): boolean {
         const { suites } = options;

packages/perf-insight/src/runBenchmarkCli.mts

@@ -9,9 +9,16 @@ import { parseArgs } from 'node:util';
 
 import { runBenchmarkSuites } from './run.mjs';
 
-const cwdUrl = pathToFileURL(process.cwd() + '/');
+interface RunResult {
+    error?: Error;
+    /**
+     * Indicates if there were any failures in the benchmark suites.
+     * This is not an error, since nothing went wrong in the insight code.
+     */
+    hadFailures: boolean;
+}
 
-async function run(args: string[]) {
+export async function run(args: string[]): Promise<RunResult> {
     const parseConfig = {
         args,
         strict: true,
@@ -22,16 +29,18 @@ async function run(args: string[]) {
             test: { type: 'string', short: 'T', multiple: true },
             suite: { type: 'string', short: 'S', multiple: true },
             register: { type: 'string', multiple: true },
+            root: { type: 'string' },
         },
     } as const satisfies ParseArgsConfig;
 
     const parsed = parseArgs(parseConfig);
+    const cwdUrl = parsed.values.root ? new URL(parsed.values.root) : pathToFileURL(process.cwd() + '/');
 
     const repeat = Number(parsed.values['repeat'] || '0') || undefined;
     const timeout = Number(parsed.values['timeout'] || '0') || undefined;
     const tests = parsed.values['test'];
     const suites = parsed.values['suite'];
-    await registerLoaders(parsed.values['register']);
+    await registerLoaders(parsed.values['register'], cwdUrl);
 
     const errors: Error[] = [];
 
@@ -54,13 +63,21 @@ async function run(args: string[]) {
         console.error('Errors:');
         errors.forEach((err) => console.error('- %s\n%o', err.message, err.cause));
         process.exitCode = 1;
-        return;
+        return { error: errors[0], hadFailures: true };
     }
 
-    await runBenchmarkSuites(undefined, { repeat, timeout, tests, suites });
+    try {
+        const r = await runBenchmarkSuites(undefined, { repeat, timeout, tests, suites });
+        process.exitCode = process.exitCode || (r.hadFailures ? 1 : 0);
+        return { error: errors[0], hadFailures: r.hadFailures };
+    } catch (e) {
+        // console.error('Failed to run benchmark suites.', e);
+        process.exitCode = 1;
+        return { error: e as Error, hadFailures: true };
+    }
 }
 
-async function registerLoaders(loaders: string[] | undefined) {
+async function registerLoaders(loaders: string[] | undefined, cwdUrl: URL) {
     if (!loaders?.length) return;
 
     const module = await import('module');
@@ -79,4 +96,10 @@ async function registerLoaders(loaders: string[] | undefined) {
     loaders.forEach(registerLoader);
 }
 
-run(process.argv.slice(2));
+// console.error('args: %o', process.argv);
+
+if ((process.argv[1] ?? '').endsWith('runBenchmarkCli.mjs')) {
+    run(process.argv.slice(2)).then((result) => {
+        if (result.error) process.exitCode = 1;
+    });
+}

packages/perf-insight/src/runBenchmarkCli.test.mts

@@ -0,0 +1,21 @@
+import { describe, expect, test } from 'vitest';
+
+import { run } from './runBenchmarkCli.mjs';
+
+describe('runBenchmarkCli', () => {
+    test.each`
+        file                         | args                    | root
+        ${'src/exampleMap.perf.mts'} | ${'--suite map -t 500'} | ${'../../../examples/'}
+    `('runBenchmarkCli $file $args', async ({ file, args, root }) => {
+        expect(run).toBeTypeOf('function');
+
+        args = typeof args === 'string' ? args.split(/\s+/g) : args;
+        const r = new URL(root, import.meta.url).href;
+        const fileUrl = new URL(file, r).href;
+
+        await expect(run([fileUrl, '--root', r, ...args])).resolves.toEqual({
+            error: undefined,
+            hadFailures: true,
+        });
+    });
+});