Automation: main-next integrate

.changeset/lazy-coats-bake.md

@@ -0,0 +1,13 @@
+---
+"@fluidframework/container-definitions": minor
+"@fluidframework/container-loader": minor
+"@fluid-experimental/tree": minor
+"@fluid-experimental/tree2": minor
+---
+
+Move closeAndGetPendingLocalState to IContainerExperimental
+
+This change deprecates the experimental method closeAndGetPendingLocalState on IContainer and moves it to IContainerExperimental.
+IContainerExperimental is an interface that is easily casted to, which enables partners to access experimental features for testing and evaluation.
+Moving the experimental method off IContainer will reduce exposure and churn on that production interface as we iterate on and finalize our experimental features.
+Experimental features should not be used in production environments.

.vscode/launch.json

@@ -95,7 +95,16 @@
 			"request": "launch",
 			"skipFiles": ["<node_internals>/**"],
 			"type": "node",
-			"args": ["exec", "--releaseGroup", "build-tools", "--no-private", "--", "pwd"],
+		},
+		{
+			// Runs the `flub test-only-filter` command with a debugger attached.
+			"name": "flub test-only-filter",
+			"program": "${workspaceFolder}/build-tools/packages/build-cli/bin/dev",
+			"cwd": "${workspaceFolder}/build-tools/packages/build-cli",
+			"request": "launch",
+			"skipFiles": ["<node_internals>/**"],
+			"type": "node",
+			"args": ["test-only-filter", "--json", "--all"],
 		},
 		{
 			"name": "flub bump deps",

UPCOMING.md

@@ -5,3 +5,7 @@
 ## IDeltaManager members disposed and dispose() deprecated
 
 Directly calling dispose() on the IDeltaManager puts the system in an inconsistent state, and inspecting the disposed state of the IDeltaManager is not recommended (instead, prefer to inspect either the IContainer.disposed, IContainerRuntime.disposed, or IFluidDataStoreRuntime.disposed depending on your scenario). These members have been deprecated from the interface and will be removed in an upcoming release.
+
+## Move closeAndGetPendingLocalState to IContainerExperimental
+
+This change deprecates the experimental method closeAndGetPendingLocalState on IContainer and moves it to IContainerExperimental. IContainerExperimental is an interface that is easily casted to, which enables partners to access experimental features for testing and evaluation. Moving the experimental method off IContainer will reduce exposure and churn on that production interface as we iterate on and finalize our experimental features. Experimental features should not be used in production environments.

api-report/container-definitions.api.md

@@ -120,6 +120,7 @@ export interface IContainer extends IEventProvider<IContainerEvents>, IFluidRout
     readonly audience: IAudience;
     readonly clientId?: string | undefined;
     close(error?: ICriticalContainerError): void;
+    // @deprecated (undocumented)
     closeAndGetPendingLocalState(): string;
     readonly closed: boolean;
     connect(): void;

api-report/container-loader.api.md

@@ -40,9 +40,10 @@ export interface ICodeDetailsLoader extends Partial<IProvideFluidCodeDetailsComp
     load(source: IFluidCodeDetails): Promise<IFluidModuleWithDetails>;
 }
 
-// @internal
+// @public
 export interface IContainerExperimental extends IContainer {
-    getPendingLocalState(): string;
+    closeAndGetPendingLocalState(): string;
+    getPendingLocalState?(): string;
 }
 
 // @public

api-report/tree2.api.md

@@ -1808,7 +1808,7 @@ export interface SequenceFieldEditBuilder {
     delete(index: number, count: number): void;
     insert(index: number, newContent: ITreeCursor | readonly ITreeCursor[]): void;
     move(sourceIndex: number, count: number, destIndex: number): void;
-    revive(index: number, count: number, detachedBy: RevisionTag, reviver: NodeReviver, detachIndex: number, isIntention?: true): void;
+    revive(index: number, count: number, detachedBy: RevisionTag, detachId: ChangesetLocalId, reviver: NodeReviver, isIntention?: true): void;
 }
 
 // @alpha

build-tools/packages/build-cli/.eslintrc.cjs

@@ -47,6 +47,9 @@ module.exports = {
 		// Causes issues with some versions of node
 		"unicorn/prefer-node-protocol": "off",
 
+		// Too strict for our needs
+		"unicorn/filename-case": "off",
+
 		// Deprecated in 2018: https://eslint.org/blog/2018/11/jsdoc-end-of-life/
 		"valid-jsdoc": "off",
 	},

build-tools/packages/build-cli/README.md

@@ -46,6 +46,7 @@ USAGE
 * [`flub merge`](docs/merge.md) - Sync branches depending on the batch size passed
 * [`flub release`](docs/release.md) - Release commands are used to manage the Fluid release process.
 * [`flub run`](docs/run.md) - Generate a report from input bundle stats collected through the collect bundleStats command.
+* [`flub test-only-filter`](docs/test-only-filter.md) - FOR INTERNAL TESTING ONLY. This command is used only to test the common package filtering and selection logic that is used across the CLI. FOR INTERNAL TESTING ONLY.
 * [`flub typetests`](docs/typetests.md) - Updates configuration for type tests in package.json files. If the previous version changes after running preparation, then npm install must be run before building.
 
 <!-- commandsstop -->

build-tools/packages/build-cli/api-report/build-cli.api.md

@@ -7,7 +7,10 @@
 import { run } from '@oclif/core';
 
 // @internal
-export type ReleaseGroup = string;
+export const knownReleaseGroups: readonly ["build-tools", "client", "server", "gitrest", "historian"];
+
+// @internal
+export type ReleaseGroup = typeof knownReleaseGroups[number] | string;
 
 // @internal
 export type ReleasePackage = string;

build-tools/packages/build-cli/docs/test-only-filter.md

@@ -0,0 +1,60 @@
+`flub test-only-filter`
+=======================
+
+FOR INTERNAL TESTING ONLY. This command is used only to test the common package filtering and selection logic that is used across the CLI. FOR INTERNAL TESTING ONLY.
+
+* [`flub test-only-filter`](#flub-test-only-filter)
+
+## `flub test-only-filter`
+
+FOR INTERNAL TESTING ONLY. This command is used only to test the common package filtering and selection logic that is used across the CLI. FOR INTERNAL TESTING ONLY.
+
+```
+USAGE
+  $ flub test-only-filter [-v | --quiet] [--concurrency <value>] [--all | --dir <value> | --packages | -g
+    client|server|azure|build-tools|gitrest|historian|all | --releaseGroupRoot
+    client|server|azure|build-tools|gitrest|historian|all] [--private] [--scope <value> | --skipScope <value>] [--json]
+
+FLAGS
+  --concurrency=<value>  [default: 25] The number of tasks to execute concurrently.
+
+PACKAGE SELECTION FLAGS
+  -g, --releaseGroup=<option>...  Run on all child packages within the specified release groups. This does not include
+                                  release group root packages. To include those, use the --releaseGroupRoot argument.
+                                  Cannot be used with --all, --dir, or --packages.
+                                  <options: client|server|azure|build-tools|gitrest|historian|all>
+  --all                           Run on all packages and release groups. Cannot be used with --all, --dir,
+                                  --releaseGroup, or --releaseGroupRoot.
+  --dir=<value>                   Run on the package in this directory. Cannot be used with --all, --dir,
+                                  --releaseGroup, or --releaseGroupRoot.
+  --packages                      Run on all independent packages in the repo. Cannot be used with --all, --dir,
+                                  --releaseGroup, or --releaseGroupRoot.
+  --releaseGroupRoot=<option>...  Run on the root package of the specified release groups. This does not include any
+                                  child packages within the release group. To include those, use the --releaseGroup
+                                  argument. Cannot be used with --all, --dir, or --packages.
+                                  <options: client|server|azure|build-tools|gitrest|historian|all>
+
+LOGGING FLAGS
+  -v, --verbose  Enable verbose logging.
+  --quiet        Disable all logging.
+
+GLOBAL FLAGS
+  --json  Format output as json.
+
+PACKAGE FILTER FLAGS
+  --[no-]private          Only include private packages. Use --no-private to exclude private packages instead.
+  --scope=<value>...      Package scopes to filter to. If provided, only packages whose scope matches the flag will be
+                          included. Cannot be used with --skipScope.
+  --skipScope=<value>...  Package scopes to filter out. If provided, packages whose scope matches the flag will be
+                          excluded. Cannot be used with --scope.
+
+DESCRIPTION
+  FOR INTERNAL TESTING ONLY. This command is used only to test the common package filtering and selection logic that is
+  used across the CLI. FOR INTERNAL TESTING ONLY.
+
+  This command outputs JSON containing metadata about the packages selected and filtered. This output is parsed in
+  tests. While the --json flag is technically optional, it should always be passed when using this command for testing.
+  Otherwise there is no output to be checked for correctness.
+```
+
+_See code: [src/commands/test-only-filter.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/test-only-filter.ts)_

build-tools/packages/build-cli/package.json

@@ -23,13 +23,14 @@
 	],
 	"scripts": {
 		"build": "npm run clean:manifest && concurrently npm:build:compile npm:lint && concurrently npm:build:docs npm:build:manifest npm:build:diagrams",
-		"build:compile": "npm run tsc && npm run build:copy && npm run build:readme",
+		"build:compile": "npm run tsc && npm run build:test && npm run build:copy && npm run build:readme",
 		"build:copy": "copyfiles -u 1 \"src/**/*.fsl\" lib",
 		"build:diagrams": "jssm-viz -i \"./src/machines/*.fsl\"",
 		"build:docs": "api-extractor run --local",
 		"build:machine-diagram": "jssm-viz -i \"./src/machines/*.fsl\"",
 		"build:manifest": "cross-env NODE_OPTIONS='--experimental-abortcontroller' oclif manifest",
 		"build:readme": "fluid-readme generate readme --multi",
+		"build:test": "tsc --project ./test/tsconfig.json",
 		"ci:build:docs": "api-extractor run",
 		"clean": "rimraf dist lib oclif.manifest.json *.tsbuildinfo *.build.log",
 		"clean:manifest": "rimraf oclif.manifest.json",

build-tools/packages/build-cli/src/BasePackageCommand.ts

@@ -5,6 +5,7 @@
 import { Package } from "@fluidframework/build-tools";
 import { ux, Command, Flags } from "@oclif/core";
 import async from "async";
+import { strict as assert } from "node:assert";
 
 import { BaseCommand } from "./base";
 import {
@@ -34,8 +35,23 @@ export abstract class PackageCommand<
 		...BaseCommand.flags,
 	};
 
-	private filterOptions: PackageFilterOptions | undefined;
-	private selectionOptions: PackageSelectionCriteria | undefined;
+	protected filterOptions: PackageFilterOptions | undefined;
+	protected selectionOptions: PackageSelectionCriteria | undefined;
+
+	/**
+	 * An array of packages selected based on the selection criteria.
+	 *
+	 * @remarks
+	 *
+	 * Note that these packages are not necessarily the ones that are acted on. Packages are selected, then that list is
+	 * further narrowed by filtering criteria, so this array may contain packages that are not acted on.
+	 */
+	protected selectedPackages: PackageWithKind[] | undefined;
+
+	/**
+	 * The list of packages after all filters are applied to the selected packages.
+	 */
+	protected filteredPackages: PackageWithKind[] | undefined;
 
 	/**
 	 * Called for each package that is selected/filtered based on the filter flags passed in to the command.
@@ -49,10 +65,12 @@ export abstract class PackageCommand<
 		kind: PackageKind,
 	): Promise<void>;
 
-	public async run(): Promise<void> {
+	protected parseFlags() {
 		this.selectionOptions = parsePackageSelectionFlags(this.flags);
 		this.filterOptions = parsePackageFilterFlags(this.flags);
+	}
 
+	protected async selectAndFilterPackages() {
 		if (this.selectionOptions === undefined) {
 			throw new Error(`No packages selected.`);
 		}
@@ -64,16 +82,31 @@ export abstract class PackageCommand<
 			this.filterOptions,
 		);
 
+		[this.selectedPackages, this.filteredPackages] = [selected, filtered];
+	}
+
+	public async run(): Promise<any> {
+		this.parseFlags();
+
+		assert(this.selectionOptions !== undefined, "selectionOptions is undefined");
+		assert(this.filterOptions !== undefined, "filterOptions is undefined");
+
+		await this.selectAndFilterPackages();
+
+		assert(this.selectedPackages !== undefined, "selectedPackages is undefined");
+		assert(this.filteredPackages !== undefined, "filteredPackages is undefined");
+
 		this.info(
-			`Filtered ${selected.length} packages to ${listNames(
-				filtered.map((pkg) => pkg.directory),
+			`Filtered ${this.selectedPackages.length} packages to ${listNames(
+				this.filteredPackages.map((pkg) => pkg.directory),
 			)}`,
 		);
 
-		return this.processPackages(filtered);
+		await this.processPackages(this.filteredPackages);
+		return undefined;
 	}
 
-	private async processPackages(packages: PackageWithKind[]): Promise<void> {
+	protected async processPackages(packages: PackageWithKind[]): Promise<void> {
 		let started = 0;
 		let finished = 0;
 		let succeeded = 0;
@@ -120,6 +153,10 @@ export abstract class PackageCommand<
 	}
 }
 
-function listNames(strings: string[]): string {
-	return strings.length > 10 ? `${strings.length}` : `${strings.length} (${strings.join(", ")})`;
+function listNames(strings: string[] | undefined): string {
+	return strings === undefined
+		? ""
+		: strings.length > 10
+		? `${strings.length}`
+		: `${strings.length} (${strings.join(", ")})`;
 }

build-tools/packages/build-cli/src/commands/generate/changeset.ts

@@ -15,6 +15,7 @@ import prompts from "prompts";
 import { BaseCommand } from "../../base";
 import { Repository, getDefaultBumpTypeForBranch } from "../../lib";
 import GenerateUpcomingCommand from "./upcoming";
+import { ReleaseGroup } from "../../releaseGroups";
 
 /**
  * If more than this number of packages are changed relative to the selected branch, the user will be prompted to select

build-tools/packages/build-cli/src/commands/test-only-filter.ts

@@ -0,0 +1,74 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { Package } from "@fluidframework/build-tools";
+import { strict as assert } from "node:assert";
+import { PackageCommand } from "../BasePackageCommand";
+import { PackageWithKind } from "../filter";
+
+interface FilterCommandResult {
+	selected: Pick<Package, "name" | "directory">[];
+	filtered: Pick<Package, "name" | "directory">[];
+}
+
+/**
+ * This command is used to test the common package filtering and selection logic that is used across the CLI. It
+ * subclasses PackageCommand and outputs JSON containing metadata about the packages selected and filtered. This output
+ * is parsed in tests.
+ *
+ * It may not be obvious why these command-level tests are needed in addition to testing the functions that do most of
+ * the filtering (see ../filter.test.ts). The reason is that the code to translate oclif's flags into selection/filter
+ * objects lives in the PackageCommand command. There's no way (that I know of) to mock an oclif command's flags.
+ *
+ * Instead, this test command is a real command, and the tests call it using the oclif command test infrastructure. This
+ * ensures that we are testing the complete pipeline from command flag parsing, to creating selection/filter objects
+ * from the flags, through applying to those filters to the packages in the repo.
+ *
+ * While the --json flag is technically optional, it should always be passed when using this command for testing.
+ * Otherwise there is no output to be checked for correctness.
+ */
+export default class FilterCommand extends PackageCommand<typeof FilterCommand> {
+	static summary = `FOR INTERNAL TESTING ONLY. This command is used only to test the common package filtering and selection logic that is used across the CLI. FOR INTERNAL TESTING ONLY.`;
+	static description = `This command outputs JSON containing metadata about the packages selected and filtered. This output is parsed in tests. While the --json flag is technically optional, it should always be passed when using this command for testing. Otherwise there is no output to be checked for correctness.`;
+
+	// hide the command from help since it's only supposed to be used for internal testing
+	static hidden = false;
+
+	static enableJsonFlag = true;
+
+	protected async processPackage(pkg: Package): Promise<void> {
+		// do nothing
+	}
+
+	protected async processPackages(packages: PackageWithKind[]): Promise<void> {
+		// do nothing
+	}
+
+	public async run(): Promise<FilterCommandResult> {
+		await super.run();
+
+		assert(this.selectionOptions !== undefined, "selectionOptions is undefined");
+		assert(this.filterOptions !== undefined, "filterOptions is undefined");
+		assert(this.selectedPackages !== undefined, "selectedPackages is undefined");
+		assert(this.filteredPackages !== undefined, "filteredPackages is undefined");
+
+		const context = await this.getContext();
+		const pkgs = {
+			selected: this.selectedPackages.map((p) => {
+				return {
+					name: p.name,
+					directory: context.repo.relativeToRepo(p.directory),
+				};
+			}),
+			filtered: this.filteredPackages.map((p) => {
+				return {
+					name: p.name,
+					directory: context.repo.relativeToRepo(p.directory),
+				};
+			}),
+		};
+
+		return pkgs;
+	}
+}