feat(fluid-build): Add support for declarative tasks

build-tools/packages/build-tools/package.json

@@ -50,6 +50,7 @@
 		"find-up": "^7.0.0",
 		"fs-extra": "^11.2.0",
 		"glob": "^7.2.3",
+		"globby": "^11.1.0",
 		"ignore": "^5.2.4",
 		"json5": "^2.2.3",
 		"lodash": "^4.17.21",

build-tools/packages/build-tools/src/common/utils.ts

@@ -9,14 +9,19 @@ import * as path from "path";
 import isEqual from "lodash.isequal";
 
 /**
- *	An array of commands that are known to have subcommands and should be parsed as such
+ *	An array of commands that are known to have subcommands and should be parsed as such. These will be combined with
+ *	any additional commands provided in the Fluid build config.
  */
-const multiCommandExecutables = ["flub", "biome"];
+const defaultMultiCommandExecutables = ["flub", "biome"] as const;
 
-export function getExecutableFromCommand(command: string) {
+export function getExecutableFromCommand(command: string, multiCommandExecutables: string[]) {
 	let toReturn: string;
 	const commands = command.split(" ");
-	if (multiCommandExecutables.includes(commands[0])) {
+	const multiExecutables: Set<string> = new Set([
+		...defaultMultiCommandExecutables,
+		...multiCommandExecutables,
+	]);
+	if (multiExecutables.has(commands[0])) {
 		// For multi-commands (e.g., "flub bump ...") our heuristic is to scan for the first argument that cannot
 		// be the name of a sub-command, such as '.' or an argument that starts with '-'.
 		//

build-tools/packages/build-tools/src/fluidBuild/fluidBuildConfig.ts

@@ -31,6 +31,19 @@ export interface IFluidBuildConfig {
 	 */
 	tasks?: TaskDefinitionsOnDisk;
 
+	/**
+	 * Add task handlers based on configuration only. This allows you to add incremental build support for executables and
+	 * commands that don't support it.
+	 */
+	declarativeTasks?: DeclarativeTasks;
+
+	/**
+	 * An array of commands that are known to have subcommands and should be parsed as such.
+	 *
+	 * These values will be combined with the default values: ["flub", "biome"]
+	 */
+	multiCommandExecutables?: string[];
+
 	/**
 	 * A mapping of package or release group names to metadata about the package or release group. This can only be
 	 * configured in the repo-wide Fluid build config (the repo-root package.json).
@@ -67,3 +80,44 @@ export type IFluidBuildDirEntry = string | IFluidBuildDir | (string | IFluidBuil
 export interface IFluidBuildDirs {
 	[name: string]: IFluidBuildDirEntry;
 }
+
+/**
+ * Declarative tasks allow fluid-build to support incremental builds for tasks it doesn't natively identify. A
+ * DeclarativeTask defines a set of input and output globs, and files matching those globs will be included in the
+ * donefiles for the task. Note that gitignored files are treated differently for input globs vs. output globs. See the
+ * property documentation for details.
+ */
+export interface DeclarativeTask {
+	/**
+	 * An array of globs that will be used to identify input files for the task. Globs are interpreted relative to the
+	 * package. **Files ignored by git are never included.**
+	 */
+	inputGlobs: string[];
+
+	/**
+	 * An array of globs that will be used to identify input files for the task. Unlike the inputGlobs, outputGlobs
+	 * **will** match files ignored by git, because build output is often gitignored.
+	 */
+	outputGlobs: string[];
+
+	/**
+	 * Configures how gitignore rules are applied. "input" applies gitignore rules to the input, "output" applies them to
+	 * the output, and including both values will apply the gitignore rules to both the input and output globs.
+	 *
+	 * The default value, `["input"]` applies gitignore rules to the input, but not the output. This is the right behavior
+	 * for many tasks since most tasks use source-controlled files as input but generate gitignored build output. However,
+	 * it can be adjusted on a per-task basis depending on the needs of the task.
+	 *
+	 * @defaultValue `["input"]`
+	 */
+	gitignore?: ("input" | "output")[];
+}
+
+/**
+ * This mapping of executable/command name to DeclarativeTask is used to connect the task to the correct executable(s).
+ * Note that multi-command executables must also be included in the multiCommandExecutables setting. If they are not,
+ * the commands will not be parsed correctly and may not match the task as expected.
+ */
+export interface DeclarativeTasks {
+	[executable: string]: DeclarativeTask;
+}

build-tools/packages/build-tools/src/fluidBuild/tasks/leaf/declarativeTask.ts

@@ -0,0 +1,80 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import globby from "globby";
+
+import type { BuildContext } from "../../buildContext";
+import type { BuildPackage } from "../../buildGraph";
+import type { DeclarativeTask } from "../../fluidBuildConfig";
+import type { TaskHandlerFunction } from "../taskHandlers";
+import { LeafTask, LeafWithFileStatDoneFileTask } from "./leafTask";
+
+class DeclarativeTaskHandler extends LeafWithFileStatDoneFileTask {
+	constructor(
+		node: BuildPackage,
+		command: string,
+		context: BuildContext,
+		taskName: string | undefined,
+		private readonly taskDefinition: DeclarativeTask,
+	) {
+		super(node, command, context, taskName);
+	}
+
+	/**
+	 * Use hashes instead of modified times in donefile.
+	 */
+	protected get useHashes(): boolean {
+		return true;
+	}
+
+	protected async getInputFiles(): Promise<string[]> {
+		const { inputGlobs, gitignore: gitignoreSetting } = this.taskDefinition;
+
+		// Ignore gitignored files if the setting is undefined, since the default is ["input"]. Otherwise check that it
+		// includes "input".
+		const gitignore: boolean =
+			gitignoreSetting === undefined || gitignoreSetting.indexOf("input") !== -1;
+		const inputFiles = await globby(inputGlobs, {
+			cwd: this.node.pkg.directory,
+			// file paths returned from getInputFiles and getOutputFiles should always be absolute
+			absolute: true,
+			gitignore,
+		});
+		return inputFiles;
+	}
+
+	protected async getOutputFiles(): Promise<string[]> {
+		const { outputGlobs, gitignore: gitignoreSetting } = this.taskDefinition;
+
+		const gitignore: boolean = gitignoreSetting?.indexOf("output") !== -1;
+		const outputFiles = await globby(outputGlobs, {
+			cwd: this.node.pkg.directory,
+			// file paths returned from getInputFiles and getOutputFiles should always be absolute
+			absolute: true,
+			gitignore,
+		});
+		return outputFiles;
+	}
+}
+
+/**
+ * Generates a task handler for a declarative task dynamically.
+ *
+ * @param taskDefinition - The declarative task definition.
+ * @returns a function that can be used to instantiate a LeafTask to handle a task.
+ */
+export function createDeclarativeTaskHandler(
+	taskDefinition: DeclarativeTask,
+): TaskHandlerFunction {
+	const handler: TaskHandlerFunction = (
+		node: BuildPackage,
+		command: string,
+		context: BuildContext,
+		taskName?: string,
+	): LeafTask => {
+		return new DeclarativeTaskHandler(node, command, context, taskName, taskDefinition);
+	};
+	return handler;
+}

build-tools/packages/build-tools/src/fluidBuild/tasks/leaf/leafTask.ts

@@ -153,7 +153,10 @@ export abstract class LeafTask extends Task {
 	}
 
 	public get executable() {
-		return getExecutableFromCommand(this.command);
+		return getExecutableFromCommand(
+			this.command,
+			this.context.fluidBuildConfig?.multiCommandExecutables ?? [],
+		);
 	}
 
 	protected get useWorker() {

build-tools/packages/build-tools/src/fluidBuild/tasks/taskFactory.ts

@@ -9,9 +9,10 @@ import { BuildPackage } from "../buildGraph";
 import { GroupTask } from "./groupTask";
 import { ApiExtractorTask } from "./leaf/apiExtractorTask";
 import { BiomeTask } from "./leaf/biomeTasks";
+import { createDeclarativeTaskHandler } from "./leaf/declarativeTask";
 import { FlubCheckLayerTask, FlubCheckPolicyTask, FlubListTask } from "./leaf/flubTasks";
 import { GenerateEntrypointsTask } from "./leaf/generateEntrypointsTask.js";
-import { type LeafTask, UnknownLeafTask } from "./leaf/leafTask";
+import { UnknownLeafTask } from "./leaf/leafTask";
 import { EsLintTask, TsLintTask } from "./leaf/lintTasks";
 import {
 	CopyfilesTask,
@@ -28,15 +29,11 @@ import { Ts2EsmTask } from "./leaf/ts2EsmTask";
 import { TscMultiTask, TscTask } from "./leaf/tscTask";
 import { WebpackTask } from "./leaf/webpackTask";
 import { Task } from "./task";
+import { type TaskHandler, isConstructorFunction } from "./taskHandlers";
 
 // Map of executable name to LeafTasks
 const executableToLeafTask: {
-	[key: string]: new (
-		node: BuildPackage,
-		command: string,
-		context: BuildContext,
-		taskName?: string,
-	) => LeafTask;
+	[key: string]: TaskHandler;
 } = {
 	"ts2esm": Ts2EsmTask,
 	"tsc": TscTask,
@@ -75,7 +72,32 @@ const executableToLeafTask: {
 	// pipeline then this mapping will have to be updated.
 	"renamer": RenameTypesTask,
 	"flub rename-types": RenameTypesTask,
-};
+} as const;
+
+/**
+ * Given a command executable, attempts to find a matching TaskHandler that will handle the task. If one is found, it is
+ * returned; otherwise, it returns `UnknownLeafTask` as the default handler.
+ *
+ * The built-in executableToLeafTask constant is checked first, followed by any DeclarativeTasks that are defined in the
+ * fluid-build config.
+ *
+ * @param executable The command executable to find a matching task handler for.
+ * @returns A TaskHandler for the task, if found. Otherwise `UnknownLeafTask` as the default handler.
+ */
+function getTaskForExecutable(executable: string, context: BuildContext): TaskHandler {
+	const found: TaskHandler | undefined = executableToLeafTask[executable];
+	if (found === undefined) {
+		const config = context.fluidBuildConfig;
+		const declarativeTasks = config?.declarativeTasks;
+		const taskMatch = declarativeTasks?.[executable];
+		if (taskMatch !== undefined) {
+			return createDeclarativeTaskHandler(taskMatch);
+		}
+	}
+
+	// If no handler is found, return the UnknownLeafTask as the default handler.
+	return found ?? UnknownLeafTask;
+}
 
 /**
  * Regular expression to parse `concurrently` arguments that specify package scripts.
@@ -166,13 +188,21 @@ export class TaskFactory {
 			return new GroupTask(node, command, context, [subTask], taskName);
 		}
 
-		// Leaf task
-		const executable = getExecutableFromCommand(command).toLowerCase();
-		const ctor = executableToLeafTask[executable];
-		if (ctor) {
-			return new ctor(node, command, context, taskName);
+		// Leaf tasks; map the executable to a known task type. If none is found, the UnknownLeafTask is used.
+		const executable = getExecutableFromCommand(
+			command,
+			context.fluidBuildConfig?.multiCommandExecutables ?? [],
+		).toLowerCase();
+
+		// Will return a task-specific handler or the UnknownLeafTask
+		const handler = getTaskForExecutable(executable, context);
+
+		// Invoke the function or constructor to create the task handler
+		if (isConstructorFunction(handler)) {
+			return new handler(node, command, context, taskName);
+		} else {
+			return handler(node, command, context, taskName);
 		}
-		return new UnknownLeafTask(node, command, context, taskName);
 	}
 
 	/**

build-tools/packages/build-tools/src/fluidBuild/tasks/taskHandlers.ts

@@ -0,0 +1,44 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { BuildContext } from "../buildContext";
+import type { BuildPackage } from "../buildGraph";
+import type { LeafTask } from "./leaf/leafTask";
+
+/**
+ * The definition of a free function that returns a LeafTask subclass.
+ */
+export type TaskHandlerFunction = (
+	node: BuildPackage,
+	command: string,
+	context: BuildContext,
+	taskName?: string,
+) => LeafTask;
+
+/**
+ * The definition of a constructor function that returns a LeafTask subclass.
+ */
+export type TaskHandlerConstructor = new (
+	node: BuildPackage,
+	command: string,
+	context: BuildContext,
+	taskName?: string,
+) => LeafTask;
+
+/**
+ * A TaskHandler is a function that can be used to generate a `LeafTask` that will handle a particular fluid-build task.
+ * The function can either be a constructor for a `LeafTask` subclass, or it can be a free function that returns a
+ * `LeafTask`.
+ */
+export type TaskHandler = TaskHandlerConstructor | TaskHandlerFunction;
+
+/**
+ * Type guard to check if a TaskHandler is a constructor function.
+ */
+export function isConstructorFunction(
+	handler: TaskHandler,
+): handler is TaskHandlerConstructor {
+	return typeof handler === "function" && !!handler.prototype;
+}