feat(turbo): support extends on turbo.json and detecting turbo to p…

…revent re-invocation (#152)

#### What this PR does / why we need it:

@levaintech maintains an internal fork of sticky, this PR syncs the
quality of life update that were introduced to turbo.

- feat(sticky-turbo): support `extends: [//]` on turbo.json, see
https://turbo.build/repo/docs/core-concepts/monorepos/configuring-workspaces
- feat(sticky-jest-turbo): detect turbo is the current process to
prevent re-invocation—this prevent turbo from running when the entire
process is triggered from turbo.
- chore(*): fix wrong lint script and `tsconfig.json`

#### Additional comments?:

This PR contains breaking changes when `sticky-turbo` is used
programmatically.

packages/sticky-jest/package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "build": "tsc -b ./tsconfig.build.json",
     "clean": "rm -rf dist",
-    "lint": "eslint src",
+    "lint": "eslint .",
     "test": "jest"
   },
   "eslintConfig": {

packages/sticky-jest/tsconfig.build.json

@@ -1,4 +1,9 @@
 {
   "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"],
   "exclude": ["**/*.unit.ts"]
 }

packages/sticky-jest/tsconfig.json

@@ -1,8 +1,6 @@
 {
-  "extends": "@birthdayresearch/sticky-typescript/tsconfig.json",
+  "extends": "@birthdayresearch/sticky-typescript",
   "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./dist"
-  },
-  "include": ["src"]
+    "rootDir": "./"
+  }
 }

packages/sticky-testcontainers/tsconfig.json

@@ -1,8 +1,6 @@
 {
-  "extends": "@birthdayresearch/sticky-typescript/tsconfig.json",
+  "extends": "@birthdayresearch/sticky-typescript",
   "compilerOptions": {
-    "rootDir": "./",
-    "outDir": "./dist"
-  },
-  "include": ["src"]
+    "rootDir": "./"
+  }
 }

packages/sticky-turbo-jest/jest-turbo.js

@@ -1,41 +1,24 @@
-const { spawnSync } = require('node:child_process');
-const { existsSync } = require('node:fs');
-const { join } = require('node:path');
-const { TurboJson, PackageJson } = require('@birthdayresearch/sticky-turbo');
+const { Turbo } = require('@birthdayresearch/sticky-turbo');
 
 module.exports = async function (_, project) {
-  const turboJson = new TurboJson(project.rootDir);
-  const packageJson = new PackageJson(project.rootDir);
-  const displayName = project.displayName.name;
-
-  for (const script of turboJson.getPipeline(displayName)?.dependsOn ?? []) {
-    if (script.startsWith('^')) {
-      await run(turboJson.getRootDir(), script.substring(1), `${packageJson.getName()}^...`);
-    } else if (packageJson.hasScript(script)) {
-      await run(turboJson.getRootDir(), script, packageJson.getName());
-    }
+  if (isTurbo()) {
+    console.log('jest-turbo: turbo is already running. skipping...');
+    return;
   }
-};
 
-async function run(rootDir, script, filter) {
-  const bin = './node_modules/.bin/turbo';
-  const args = ['run', script, `--filter=${filter}`, `--output-logs=new-only`];
-
-  if (!existsSync(join(rootDir, bin))) {
-    throw new Error(`Cannot find ${bin}`);
-  }
-
-  const spawn = spawnSync(bin, args, {
-    stdio: ['inherit', 'inherit', 'pipe'],
-    cwd: rootDir,
+  const turbo = new Turbo(project.rootDir);
+  // project.displayName represent the script
+  const script = project.displayName.name;
+  turbo.runBefore(script, {
+    'output-logs': 'new-only',
   });
+};
 
-  // Throw error if non-zero exit code encountered
-  if (spawn.status !== 0) {
-    const failureMessage =
-      spawn.stderr && spawn.stderr.length > 0
-        ? spawn.stderr.toString('utf-8')
-        : `Encountered non-zero exit code while running script: ${script}`;
-    throw new Error(failureMessage);
-  }
+/**
+ * Detects if the current process is a turbo invocation.
+ * If so, we don't need to run turbo again.
+ * @return {boolean}
+ */
+function isTurbo() {
+  return process.env.TURBO_INVOCATION_DIR !== undefined;
 }

packages/sticky-turbo/package.json

@@ -10,7 +10,8 @@
   "scripts": {
     "build": "tsc -b ./tsconfig.build.json",
     "clean": "rm -rf dist",
-    "lint": "eslint src"
+    "lint": "eslint .",
+    "test": "jest"
   },
   "eslintConfig": {
     "parserOptions": {
@@ -20,7 +21,13 @@
       "@birthdayresearch"
     ]
   },
+  "jest": {
+    "preset": "@birthdayresearch/sticky-jest"
+  },
   "dependencies": {
     "turbo": "1.9.8"
+  },
+  "devDependencies": {
+    "@birthdayresearch/sticky-jest": "workspace:*"
   }
 }

packages/sticky-turbo/src/Turbo.ts

@@ -0,0 +1,214 @@
+import { spawnSync, SpawnSyncReturns, StdioOptions } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+import { PackageJson } from './PackageJson';
+
+export class Turbo {
+  private readonly rootDir: string;
+
+  constructor(protected readonly cwd: string, depth = 4) {
+    const path = findRootTurboJsonPath(cwd, depth);
+    if (path === undefined) {
+      throw new Error('turbo.json not found');
+    }
+    this.rootDir = dirname(path);
+  }
+
+  getRootDir(): string {
+    return this.rootDir;
+  }
+
+  /**
+   * @param {string} task to plan
+   * @param {Record<string, string | undefined>} opts for turbo CLI
+   * @return {string[]} planned list of packages to run
+   */
+  planPackages(task: string, opts?: Record<string, string | undefined>): string[] {
+    const { tasks } = this.plan(task, opts);
+    return tasks
+      .filter((t: any) => t.task === task)
+      .filter((t: any) => t.command !== '<NONEXISTENT>')
+      .map((t: any) => t.package as string);
+  }
+
+  /**
+   * @param {string} task to run
+   * @param {Record<string, string | undefined>} opts for turbo CLI
+   * @deprecated use runTask
+   */
+  run(task: string, opts?: Record<string, string | undefined>): void {
+    this.exec(['run', task], opts);
+  }
+
+  /**
+   * @param {string} task to run
+   * @param {Record<string, string | undefined>} opts for turbo CLI
+   */
+  runTask(task: string, opts?: Record<string, string | undefined>): void {
+    this.exec(['run', task, ...optionsAsArgs(opts)]);
+  }
+
+  runTasks(
+    pipelines: {
+      task: string;
+      opts?: Record<string, string | undefined>;
+    }[],
+    opts?: Record<string, string | undefined>,
+  ): void {
+    /**
+     * Optimize the order of the pipelines to run where the most concurrent pipelines are run first.
+     */
+    const priority: Record<string, number> = {
+      'build:docker': 1,
+      default: Number.MAX_VALUE,
+    };
+    pipelines.sort((a, b): number => (priority[a.task] || priority.default) - (priority[b.task] || priority.default));
+
+    for (const pipeline of pipelines) {
+      this.exec([
+        'run',
+        pipeline.task,
+        ...optionsAsArgs({
+          ...opts,
+          ...pipeline.opts,
+        }),
+      ]);
+    }
+  }
+
+  /**
+   * By taking advantage of content-aware hashing from turborepo. `dependsOn` only runs if the pipeline `inputs` has
+   * changed.
+   *
+   * @param {string} task `dependsOn` without running the script
+   * @param {Record<string, string | undefined>} opts for turbo CLI
+   */
+  runBefore(task: string, opts?: Record<string, string | undefined>): void {
+    const packageJson = new PackageJson(this.cwd);
+    const plan = this.plan(task, {
+      ...opts,
+      only: undefined,
+      filter: packageJson.getName(),
+    });
+
+    const pipelines = plan.tasks[0].resolvedTaskDefinition.dependsOn
+      .map((dependOnScript: string) => {
+        if (dependOnScript.startsWith('^')) {
+          return {
+            task: dependOnScript.substring(1),
+            opts: {
+              filter: `${packageJson.getName()}^...`,
+            },
+          };
+        }
+        if (packageJson.hasScript(dependOnScript)) {
+          return {
+            task: dependOnScript,
+            opts: {
+              filter: packageJson.getName(),
+            },
+          };
+        }
+
+        return undefined;
+      })
+      .filter((p: any) => p !== undefined);
+
+    this.runTasks(pipelines, opts);
+  }
+
+  /**
+   * @param {string} task to plan
+   * @param {Record<string, string | undefined>} opts for turbo CLI
+   * @return {any} json object
+   */
+  private plan(task: string, opts?: Record<string, string | undefined>): any {
+    const spawn = this.exec(
+      ['run', task],
+      {
+        ...opts,
+        dry: 'json',
+      },
+      'pipe',
+    );
+
+    return JSON.parse(spawn.stdout);
+  }
+
+  /**
+   * @throws Error
+   */
+  private exec(
+    args: string[],
+    opts?: Record<string, string | undefined>,
+    stdio: StdioOptions = ['inherit', 'inherit', 'pipe'],
+  ): SpawnSyncReturns<string> {
+    const bin = './node_modules/.bin/turbo';
+
+    if (!existsSync(join(this.getRootDir(), bin))) {
+      throw new Error(`Cannot find ${bin}`);
+    }
+
+    const spawn = spawnSync(bin, [...args, ...optionsAsArgs(opts)], {
+      stdio,
+      maxBuffer: 20_000_000,
+      cwd: this.getRootDir(),
+      encoding: 'utf-8',
+    });
+
+    // throw an error if non-zero exit code encountered
+    if (spawn.status !== 0) {
+      if (spawn.stderr?.length === 0) {
+        throw new Error(`Encountered non-zero exit code while running: ${args.join(' ')}`);
+      }
+      if (spawn.stderr?.includes('error preparing engine: Could not find the following tasks in project: ')) {
+        // Allow skipping of scripts that are not found in the project.
+        // Porting back a desired behavior that was present in <1.8.0
+        // TODO: we need to hide this misleading error "Could not find the following tasks in project" in console stdout
+        // See https://github.com/vercel/turbo/pull/3828
+        return spawn;
+      }
+
+      throw new Error(spawn.stderr);
+    }
+
+    return spawn;
+  }
+}
+
+/**
+ * Find the path of root turbo.json, locates where the monorepo root is.
+ *
+ * @param cwd {string} of the current working directory
+ * @param depth {number} on how far up to search
+ */
+export function findRootTurboJsonPath(cwd: string, depth: number = 4): string | undefined {
+  const paths = cwd.split('/');
+
+  for (let i = 0; i < depth; i += 1) {
+    const path = `${paths.join('/')}/turbo.json`;
+    if (existsSync(path)) {
+      const object = JSON.parse(readFileSync(path, { encoding: 'utf-8' }));
+      if (!object.extends?.includes('//')) {
+        return path;
+      }
+    }
+    paths.pop();
+  }
+
+  return undefined;
+}
+
+function optionsAsArgs(options?: Record<string, string | undefined>): string[] {
+  if (!options) {
+    return [];
+  }
+
+  return Object.entries(options).map(([key, value]) => {
+    if (value === undefined) {
+      return `--${key}`;
+    }
+    return `--${key}=${value}`;
+  });
+}

packages/sticky-turbo/src/Turbo.unit.ts

@@ -0,0 +1,41 @@
+import * as process from 'process';
+
+import { Turbo } from './Turbo';
+
+it(`turbo.runTask('invalid-script') should be ignored; porting a desired behavior that was present in <1.8.0`, async () => {
+  const turbo = new Turbo(process.cwd());
+  turbo.runTask('invalid-script', {
+    filter: `@levain-dev/sticky-turbo^...`,
+  });
+});
+
+it(`turbo.runTasks('invalid-script') should be ignored; porting a desired behavior that was present in <1.8.0`, async () => {
+  const turbo = new Turbo(process.cwd());
+  turbo.runTasks([
+    {
+      task: 'invalid-script',
+      opts: {
+        filter: `@levain-dev/sticky-turbo^...`,
+      },
+    },
+  ]);
+});
+
+it(`should turbo.runBefore('build') without failure`, async () => {
+  const turbo = new Turbo(process.cwd());
+  turbo.runBefore('build', {
+    dry: 'json',
+  });
+});
+
+it(`turbo.planPackages('test') should generate a list of packages to run`, async () => {
+  const turbo = new Turbo(process.cwd());
+  const packages = turbo.planPackages('test');
+  expect(packages).toStrictEqual(
+    expect.arrayContaining([
+      '@birthdayresearch/sticky-jest',
+      '@birthdayresearch/sticky-testcontainers',
+      '@birthdayresearch/sticky-turbo',
+    ]),
+  );
+});