feat(plugins): use correct URI when defining bench in separate files

packages/benchmark.js-plugin/package.json

@@ -22,13 +22,15 @@
     "@babel/preset-env": "^7.22.5",
     "@types/benchmark": "^2.1.2",
     "@types/find-up": "^4.0.0",
+    "@types/lodash": "^4.14.195",
     "@types/stack-trace": "^0.0.30",
     "benchmark": "^2.1.4",
     "jest-mock-extended": "^3.0.4"
   },
   "dependencies": {
     "@codspeed/core": "workspace:^1.1.0",
     "find-up": "^6.3.0",
+    "lodash": "^4.17.10",
     "stack-trace": "1.0.0-pre2"
   },
   "peerDependencies": {

packages/benchmark.js-plugin/src/__tests__/buildSuiteAdd.test.ts

@@ -0,0 +1,72 @@
+import { Suite } from "benchmark";
+import buildSuiteAdd from "../buildSuiteAdd";
+
+const emptyBench = () => {
+  return;
+};
+
+describe("buildSuiteAdd", () => {
+  let suite: Suite;
+  let uriMap: Map<string, string>;
+
+  beforeEach(() => {
+    suite = new Suite();
+    uriMap = new Map<string, string>();
+  });
+
+  it("should register benchmark name with uriMap", () => {
+    const add = buildSuiteAdd(suite, uriMap);
+    add("test", emptyBench);
+    expect(uriMap.get("test")).toBeDefined();
+  });
+
+  it("should register benchmark name with uriMap when suite name is defined", () => {
+    suite.name = "suite";
+    const add = buildSuiteAdd(suite, uriMap);
+    add("test", emptyBench);
+    expect(uriMap.get("test")).toContain("suite");
+  });
+
+  it("should register benchmark name with uriMap when options.name is defined", () => {
+    const add = buildSuiteAdd(suite, uriMap);
+    add(emptyBench, { name: "test" });
+    expect(uriMap.get("test")).toBeDefined();
+  });
+
+  it("should call rawAdd with options object", () => {
+    const rawAdd = jest.fn();
+    suite.add = rawAdd;
+    const add = buildSuiteAdd(suite, uriMap);
+    const options = { name: "test", delay: 100 };
+    add(options);
+    expect(rawAdd).toHaveBeenCalledWith(options);
+  });
+
+  it("should call rawAdd with function and options object", () => {
+    const rawAdd = jest.fn();
+    suite.add = rawAdd;
+    const add = buildSuiteAdd(suite, uriMap);
+    const fn = emptyBench;
+    const options = { name: "test", delay: 100 };
+    add("test", fn, options);
+    expect(rawAdd).toHaveBeenCalledWith("test", fn, options);
+  });
+
+  it("should call rawAdd with name and options object", () => {
+    const rawAdd = jest.fn();
+    suite.add = rawAdd;
+    const add = buildSuiteAdd(suite, uriMap);
+    const options = { name: "test", delay: 100 };
+    add("test", options);
+    expect(rawAdd).toHaveBeenCalledWith("test", options);
+  });
+
+  it("should call rawAdd with function and undefined options", () => {
+    const rawAdd = jest.fn();
+    suite.add = rawAdd;
+    const add = buildSuiteAdd(suite, uriMap);
+    const fn = emptyBench;
+    add("test", fn);
+    expect(rawAdd).toHaveBeenCalledWith("test", fn, undefined);
+  });
+});

packages/benchmark.js-plugin/src/buildSuiteAdd.ts

@@ -0,0 +1,69 @@
+import { Options, Suite } from "benchmark";
+import { isFunction, isPlainObject } from "lodash";
+import getCallingFile from "./getCallingFile";
+
+function isOptions(options: unknown): options is Options {
+  return isPlainObject(options);
+}
+
+export default function buildSuiteAdd(
+  suite: Suite,
+  uriMap: Map<string, string>
+) {
+  const rawAdd = suite.add;
+  const suiteName = suite.name;
+
+  function registerBenchmarkName(name: string) {
+    const callingFile = getCallingFile(3);
+    let uri = callingFile;
+    if (suiteName !== undefined) {
+      uri += `::${suiteName}`;
+    }
+    uri += `::${name}`;
+    uriMap.set(name, uri);
+  }
+
+  function add(options: Options): Suite;
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  function add(fn: Function, options?: Options): Suite;
+  function add(name: string, options?: Options): Suite;
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  function add(name: string, fn: Function, options?: Options): Suite;
+  function add(name: unknown, fn?: unknown, opts?: unknown) {
+    // 1 argument: (options: Options)
+    if (isOptions(name)) {
+      if (name.name !== undefined) {
+        registerBenchmarkName(name.name);
+      }
+      return rawAdd.bind(suite)(name);
+    }
+
+    // 2 arguments: (fn: Function, options?: Options)
+    if (isFunction(name) && (isOptions(fn) || fn === undefined)) {
+      if (fn !== undefined) {
+        if (fn.name !== undefined) {
+          registerBenchmarkName(fn.name);
+        }
+      }
+      return rawAdd.bind(suite)(name, fn);
+    }
+
+    // 2 arguments: (name: string, options?: Options)
+    if (typeof name === "string" && (isOptions(fn) || fn === undefined)) {
+      registerBenchmarkName(name);
+      return rawAdd.bind(suite)(name, fn);
+    }
+
+    // 3 arguments: (name: string, fn: Function, options?: Options)
+    if (
+      typeof name === "string" &&
+      isFunction(fn) &&
+      (isOptions(opts) || opts === undefined)
+    ) {
+      registerBenchmarkName(name);
+      return rawAdd.bind(suite)(name, fn, opts);
+    }
+  }
+
+  return add;
+}

packages/benchmark.js-plugin/src/getCallingFile.ts

@@ -0,0 +1,21 @@
+import { findUpSync, Options as FindupOptions } from "find-up";
+import path, { dirname } from "path";
+import { get as getStackTrace } from "stack-trace";
+
+function getGitDir(path: string): string | undefined {
+  const dotGitPath = findUpSync(".git", {
+    cwd: path,
+    type: "directory",
+  } as FindupOptions);
+  return dotGitPath ? dirname(dotGitPath) : undefined;
+}
+
+export default function getCallingFile(depth: number): string {
+  const stack = getStackTrace();
+  const callingFile = stack[depth].getFileName();
+  const gitDir = getGitDir(callingFile);
+  if (gitDir === undefined) {
+    throw new Error("Could not find a git repository");
+  }
+  return path.relative(gitDir, callingFile);
+}

packages/benchmark.js-plugin/src/index.ts

@@ -5,9 +5,8 @@ import {
   optimizeFunctionSync,
 } from "@codspeed/core";
 import Benchmark from "benchmark";
-import { findUpSync, Options as FindupOptions } from "find-up";
-import path, { dirname } from "path";
-import { get as getStackTrace } from "stack-trace";
+import buildSuiteAdd from "./buildSuiteAdd";
+import getCallingFile from "./getCallingFile";
 
 declare const __VERSION__: string;
 
@@ -31,7 +30,7 @@ interface WithCodSpeedBenchmark
   run(options?: Benchmark.Options): Benchmark | Promise<Benchmark>;
 }
 
-interface WithCodSpeedSuite
+export interface WithCodSpeedSuite
   extends Omit<
     Benchmark.Suite,
     | "run"
@@ -93,7 +92,11 @@ function withCodSpeedBenchmark(bench: Benchmark): WithCodSpeedBenchmark {
     };
     return bench;
   }
-  const callingFile = getCallingFile();
+  const callingFile = getCallingFile(3);
+  const uriMap = new Map<string, string>();
+  if (bench.name !== undefined) {
+    uriMap.set(bench.name, `${callingFile}::${bench.name}`);
+  }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/ban-ts-comment
   // @ts-ignore
   bench.run = async function (options?: Benchmark.Options): Promise<Benchmark> {
@@ -102,6 +105,7 @@ function withCodSpeedBenchmark(bench: Benchmark): WithCodSpeedBenchmark {
       baseUri: callingFile,
       benchmarkCompletedListeners: bench.listeners("complete"),
       options,
+      uriMap,
     });
     return bench;
   };
@@ -119,7 +123,9 @@ function withCodSpeedSuite(suite: Benchmark.Suite): WithCodSpeedSuite {
     };
     return suite as WithCodSpeedSuite;
   }
-  const callingFile = getCallingFile();
+  const uriMap = new Map<string, string>();
+  suite.add = buildSuiteAdd(suite, uriMap);
+  const callingFile = getCallingFile(3);
   // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/ban-ts-comment
   // @ts-ignore
   suite.run = async function (
@@ -135,6 +141,7 @@ function withCodSpeedSuite(suite: Benchmark.Suite): WithCodSpeedSuite {
       benches,
       baseUri,
       benchmarkCompletedListeners: suite.listeners("complete"),
+      uriMap,
       options,
     });
     return suite;
@@ -148,20 +155,35 @@ interface RunBenchmarksOptions {
   benches: BenchmarkWithOptions[];
   baseUri: string;
   benchmarkCompletedListeners: CallableFunction[];
+  uriMap: Map<string, string>;
   options?: Benchmark.Options;
 }
 
+function getUri(
+  bench: BenchmarkWithOptions,
+  uriMap: Map<string, string>,
+  baseUri: string,
+  benchmarkIndex: number
+): string {
+  const name = bench.name;
+  const fallBackUri = `${baseUri}::unknown_${benchmarkIndex}`;
+  if (name === undefined) {
+    return fallBackUri;
+  }
+  return uriMap.get(name) ?? fallBackUri;
+}
+
 async function runBenchmarks({
   benches,
   baseUri,
   benchmarkCompletedListeners,
-  options,
+  uriMap,
 }: RunBenchmarksOptions): Promise<void> {
   console.log(`[CodSpeed] running with @codspeed/benchmark.js v${__VERSION__}`);
   initCore();
   for (let i = 0; i < benches.length; i++) {
     const bench = benches[i];
-    const uri = baseUri + "::" + (bench.name ?? `unknown_${i}`);
+    const uri = getUri(bench, uriMap, baseUri, i);
     const isAsync = bench.options.async || bench.options.defer;
     let benchPayload;
     if (bench.options.defer) {
@@ -193,21 +215,3 @@ async function runBenchmarks({
   }
   console.log(`[CodSpeed] Done running ${benches.length} benches.`);
 }
-
-function getCallingFile(): string {
-  const stack = getStackTrace();
-  const callingFile = stack[3].getFileName(); // [here, withCodSpeed, withCodSpeedX, actual caller]
-  const gitDir = getGitDir(callingFile);
-  if (gitDir === undefined) {
-    throw new Error("Could not find a git repository");
-  }
-  return path.relative(gitDir, callingFile);
-}
-
-function getGitDir(path: string): string | undefined {
-  const dotGitPath = findUpSync(".git", {
-    cwd: path,
-    type: "directory",
-  } as FindupOptions);
-  return dotGitPath ? dirname(dotGitPath) : undefined;
-}

packages/benchmark.js-plugin/tests/index.integ.test.ts

@@ -4,6 +4,7 @@ const mockCore = mockDeep<Measurement>();
 import type { Measurement } from "@codspeed/core";
 import Benchmark from "benchmark";
 import { withCodSpeed } from "..";
+import { registerBenchmarks } from "./registerBenchmarks";
 
 jest.mock("@codspeed/core", () => ({
   ...jest.requireActual("@codspeed/core"),
@@ -209,4 +210,16 @@ describe("Benchmark.Suite", () => {
       }
     }
   );
+  it("check nested file path is in the uri when bench is registered in another file", async () => {
+    mockCore.isInstrumented.mockReturnValue(true);
+    const suite = withCodSpeed(new Benchmark.Suite("thesuite"));
+    registerBenchmarks(suite);
+    const onComplete = jest.fn();
+    suite.on("complete", onComplete);
+    await suite.run({ maxTime: 0.1, initCount: 1 });
+    expect(mockCore.startInstrumentation).toHaveBeenCalled();
+    expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
+      "packages/benchmark.js-plugin/tests/registerBenchmarks.ts::thesuite::RegExp"
+    );
+  });
 });

packages/benchmark.js-plugin/tests/registerBenchmarks.ts

@@ -0,0 +1,11 @@
+import type { WithCodSpeedSuite } from "..";
+
+export function registerBenchmarks(suite: WithCodSpeedSuite) {
+  suite.add(
+    "RegExp",
+    function () {
+      /o/.test("Hello World!");
+    },
+    { maxTime: 0.1 }
+  );
+}

packages/tinybench-plugin/src/index.ts

@@ -19,11 +19,20 @@ export function withCodSpeed(bench: Bench): Bench {
     return bench;
   }
   initCore();
-  const callingFile = getCallingFile();
+
+  const uriMap = new Map<string, string>();
+  const rawAdd = bench.add;
+  bench.add = (name, fn, opts) => {
+    const callingFile = getCallingFile();
+    const uri = `${callingFile}::${name}`;
+    uriMap.set(name, uri);
+    return rawAdd.bind(bench)(name, fn, opts);
+  };
+  const rootCallingFile = getCallingFile();
   bench.run = async () => {
     console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`);
     for (const task of bench.tasks) {
-      const uri = callingFile + "::" + task.name;
+      const uri = uriMap.get(task.name) ?? `${rootCallingFile}::${task.name}`; // by construction, the fallback should not happen
       await optimizeFunction(task.fn);
       measurement.startInstrumentation();
       await task.fn();

packages/tinybench-plugin/tests/index.integ.test.ts

@@ -4,6 +4,7 @@ const mockCore = mockDeep<Measurement>();
 import type { Measurement } from "@codspeed/core";
 import { Bench } from "tinybench";
 import { withCodSpeed } from "..";
+import { registerBenchmarks } from "./registerBenchmarks";
 
 jest.mock("@codspeed/core", () => ({
   ...jest.requireActual("@codspeed/core"),
@@ -100,4 +101,14 @@ describe("Benchmark.Suite", () => {
       }
     }
   );
+  it("check nested file path is in the uri when bench is registered in another file", async () => {
+    mockCore.isInstrumented.mockReturnValue(true);
+    const bench = withCodSpeed(new Bench());
+    registerBenchmarks(bench);
+    await bench.run();
+    expect(mockCore.startInstrumentation).toHaveBeenCalled();
+    expect(mockCore.stopInstrumentation).toHaveBeenCalledWith(
+      "packages/tinybench-plugin/tests/registerBenchmarks.ts::RegExp"
+    );
+  });
 });

packages/tinybench-plugin/tests/registerBenchmarks.ts

@@ -0,0 +1,7 @@
+import { Bench } from "tinybench";
+
+export function registerBenchmarks(bench: Bench) {
+  bench.add("RegExp", function () {
+    /o/.test("Hello World!");
+  });
+}