Merge branch 'main' of https://github.com/microsoft/FluidFramework into ddsfuzz/rollback

Author: anthony-murphy

build-tools/packages/build-tools/src/fluidBuild/buildGraph.ts

@@ -21,6 +21,7 @@ import {
 	TaskDefinition,
 	TaskDefinitions,
 	TaskDefinitionsOnDisk,
+	TaskFileDependencies,
 	getDefaultTaskDefinition,
 	getTaskDefinitions,
 	normalizeGlobalTaskDefinitions,
@@ -158,6 +159,7 @@ export class BuildPackage {
 				before: [],
 				children: [],
 				after: [],
+				files: undefined,
 			};
 		}
 		return undefined;
@@ -172,16 +174,27 @@ export class BuildPackage {
 			this.targetTasks.set(taskName, task);
 			return task;
 		}
-		return this.createScriptTask(taskName, pendingInitDep);
+		return this.createScriptTask(taskName, pendingInitDep, config?.files);
 	}
 
-	private createScriptTask(taskName: string, pendingInitDep: Task[]) {
+	private createScriptTask(
+		taskName: string,
+		pendingInitDep: Task[],
+		files: TaskFileDependencies | undefined,
+	) {
 		const command = this.pkg.getScript(taskName);
 		if (command !== undefined && !command.startsWith("fluid-build ")) {
 			// Find the script task (without the lifecycle task)
 			let scriptTask = this.scriptTasks.get(taskName);
 			if (scriptTask === undefined) {
-				scriptTask = TaskFactory.Create(this, command, this.context, pendingInitDep, taskName);
+				scriptTask = TaskFactory.Create(
+					this,
+					command,
+					this.context,
+					pendingInitDep,
+					taskName,
+					files,
+				);
 				pendingInitDep.push(scriptTask);
 				this.tasks.push(scriptTask);
 				this.scriptTasks.set(taskName, scriptTask);
@@ -221,7 +234,14 @@ export class BuildPackage {
 			throw new Error(`${this.pkg.nameColored}: '${taskName}' must be a script task`);
 		}
 
-		const task = TaskFactory.Create(this, command, this.context, pendingInitDep, taskName);
+		const task = TaskFactory.Create(
+			this,
+			command,
+			this.context,
+			pendingInitDep,
+			taskName,
+			config?.files,
+		);
 		pendingInitDep.push(task);
 		this.tasks.push(task);
 		this.scriptTasks.set(taskName, task);
@@ -254,7 +274,7 @@ export class BuildPackage {
 			return existing;
 		}
 
-		return this.createScriptTask(taskName, pendingInitDep);
+		return this.createScriptTask(taskName, pendingInitDep, config?.files);
 	}
 
 	public getDependsOnTasks(task: Task, taskName: string, pendingInitDep: Task[]) {

build-tools/packages/build-tools/src/fluidBuild/fluidBuildConfig.ts

@@ -4,7 +4,7 @@
  */
 
 import type { InterdependencyRange } from "@fluid-tools/version-tools";
-import { TaskDefinitionsOnDisk } from "./fluidTaskDefinitions";
+import { TaskDefinitionsOnDisk, TaskFileDependencies } from "./fluidTaskDefinitions";
 
 /**
  * The version of the fluidBuild configuration currently used.
@@ -99,37 +99,7 @@ export interface IFluidBuildDirs {
  * Note that by default, gitignored files are treated differently for input globs vs. output globs. This can be
  * changed using the `gitignore` property on the task. See the documentation for that property for details.
  */
-export interface DeclarativeTask {
-	/**
-	 * An array of globs that will be used to identify input files for the task. The globs are interpreted relative to the
-	 * package the task belongs to.
-	 *
-	 * By default, inputGlobs **will not** match files ignored by git. This can be changed using the `gitignore` property
-	 * on the task. See the documentation for that property for details.
-	 */
-	inputGlobs: string[];
-
-	/**
-	 * An array of globs that will be used to identify output files for the task. The globs are interpreted relative to
-	 * the package the task belongs to.
-	 *
-	 * By default, outputGlobs **will** match files ignored by git, because build output is often gitignored. This can be
-	 *   changed using the `gitignore` property on the task. See the documentation for that property for details.
-	 */
-	outputGlobs: string[];
-
-	/**
-	 * Configures how gitignore rules are applied. "input" applies gitignore rules to the input, "output" applies them to
-	 * the output, and including both values will apply the gitignore rules to both the input and output globs.
-	 *
-	 * The default value, `["input"]` applies gitignore rules to the input, but not the output. This is the right behavior
-	 * for many tasks since most tasks use source-controlled files as input but generate gitignored build output. However,
-	 * it can be adjusted on a per-task basis depending on the needs of the task.
-	 *
-	 * @defaultValue `["input"]`
-	 */
-	gitignore?: GitIgnoreSetting;
-}
+export interface DeclarativeTask extends TaskFileDependencies {}
 
 export type GitIgnoreSetting = ("input" | "output")[];
 

build-tools/packages/build-tools/src/fluidBuild/fluidTaskDefinitions.ts

@@ -46,6 +46,50 @@ type TaskDependency =
 
 export type TaskDependencies = readonly TaskDependency[];
 
+export type GitIgnoreSetting = ("input" | "output")[];
+export interface TaskFileDependencies {
+	/**
+	 * An array of globs that will be used to identify input files for the task. The globs are interpreted relative to the
+	 * package the task belongs to.
+	 *
+	 * By default, inputGlobs **will not** match files ignored by git. This can be changed using the `gitignore` property
+	 * on the task. See the documentation for that property for details.
+	 */
+	inputGlobs: readonly string[];
+
+	/**
+	 * An array of globs that will be used to identify output files for the task. The globs are interpreted relative to
+	 * the package the task belongs to.
+	 *
+	 * By default, outputGlobs **will** match files ignored by git, because build output is often gitignored. This can be
+	 *   changed using the `gitignore` property on the task. See the documentation for that property for details.
+	 */
+	outputGlobs: readonly string[];
+
+	/**
+	 * Configures how gitignore rules are applied. "input" applies gitignore rules to the input, "output" applies them to
+	 * the output, and including both values will apply the gitignore rules to both the input and output globs.
+	 *
+	 * The default value, `["input"]` applies gitignore rules to the input, but not the output. This is the right behavior
+	 * for many tasks since most tasks use source-controlled files as input but generate gitignored build output. However,
+	 * it can be adjusted on a per-task basis depending on the needs of the task.
+	 *
+	 * @defaultValue `["input"]`
+	 */
+	gitignore?: GitIgnoreSetting;
+
+	/**
+	 * Specify whether the task will depend on the package/workspace lock file, this task will be rebuilt if the lock file
+	 * is changed. Provides an economical but broad way to ensure rebuild when tools or package dependencies changes.
+	 *
+	 * Default is true, and it is equivalent to putting the lock file in the `inputGlobs`.
+	 *
+	 * For fine-grained control, use `inputGlobs` and `outputGlobs` to specify the dependencies in `node_modules`
+	 * and set this to false.
+	 */
+	includeLockFiles?: boolean;
+}
+
 export interface TaskConfig {
 	/**
 	 * Task dependencies as a plain string array. Matched task will be scheduled to run before the current task.
@@ -93,6 +137,14 @@ export interface TaskConfig {
 	 * It can be used as an alias to a group of tasks.
 	 */
 	readonly script: boolean;
+
+	/**
+	 * For leaf tasks (i.e. tasks that have a single executable command).
+	 * Specify the input/output files the task depends on for incremental check.
+	 * Can used for unknown executable isn't a known tool (i.e. non-incremental), or to override behavior of known tools.
+	 * Error if this is specified for a non-leaf task (e.g. npm run, concurrently, multiple command with '&&').
+	 */
+	readonly files: TaskFileDependencies | undefined;
 }
 
 /**
@@ -125,14 +177,22 @@ const makeClonedOrEmptyArray = <T>(value: readonly T[] | undefined): T[] =>
  */
 function getFullTaskConfig(config: TaskConfigOnDisk): MutableTaskConfig {
 	if (isTaskDependencies(config)) {
-		return { dependsOn: [...config], script: true, before: [], children: [], after: [] };
+		return {
+			dependsOn: [...config],
+			script: true,
+			before: [],
+			children: [],
+			after: [],
+			files: undefined,
+		};
 	} else {
 		return {
 			dependsOn: makeClonedOrEmptyArray(config.dependsOn),
 			script: config.script ?? true,
 			before: makeClonedOrEmptyArray(config.before),
 			children: [],
 			after: makeClonedOrEmptyArray(config.after),
+			files: structuredClone(config.files),
 		};
 	}
 }
@@ -173,13 +233,15 @@ const defaultTaskDefinition = {
 	children: [],
 	after: ["^*"], // TODO: include "*" so the user configured task will run first, but we need to make sure it doesn't cause circular dependency first
 	isDefault: true, // only propagate to unnamed sub tasks if it is a group task
+	files: undefined,
 } as const satisfies TaskDefinition;
 const defaultCleanTaskDefinition = {
 	dependsOn: [],
 	script: true,
 	before: ["*"], // clean are ran before all the tasks, add a week dependency.
 	children: [],
 	after: [],
+	files: undefined,
 } as const satisfies TaskDefinition;
 
 const detectInvalid = (
@@ -213,6 +275,9 @@ export function normalizeGlobalTaskDefinitions(
 						`Non-script global task definition '${name}' cannot have 'before' or 'after'`,
 					);
 				}
+				if (full.files !== undefined) {
+					throw new Error(`Non-script global task definition '${name}' cannot have 'files'`);
+				}
 			}
 			detectInvalid(
 				full.dependsOn,
@@ -235,13 +300,30 @@ export function normalizeGlobalTaskDefinitions(
 				"after",
 				true,
 			);
+			if (full.files !== undefined) {
+				// "..." not allowed because the global config inherits from nothing.
+				detectInvalid(
+					full.files.inputGlobs,
+					(value) => value === "...",
+					name,
+					"files.inputGlobs",
+					true,
+				);
+				detectInvalid(
+					full.files.outputGlobs,
+					(value) => value === "...",
+					name,
+					"files.outputGlobs",
+					true,
+				);
+			}
 			taskDefinitions[name] = full;
 		}
 	}
 	return taskDefinitions;
 }
 
-function expandDotDotDot(config: readonly string[], inherited: readonly string[]) {
+function expandDotDotDot(config: readonly string[], inherited?: readonly string[]) {
 	const expanded = config.filter((value) => value !== "...");
 	if (inherited !== undefined && expanded.length !== config.length) {
 		return expanded.concat(inherited);
@@ -327,6 +409,7 @@ export function getTaskDefinitions(
 			// `children` are not inherited from the global task definitions (which should always be empty anyway)
 			children: [],
 			after: globalTaskDefinition.after.filter(globalAllowExpansionsStar),
+			files: globalTaskDefinition.files,
 		};
 	}
 
@@ -340,20 +423,37 @@ export function getTaskDefinitions(
 				if (script === undefined) {
 					throw new Error(`Script not found for task definition '${name}'`);
 				} else if (script.startsWith("fluid-build ")) {
-					throw new Error(`Script task should not invoke 'fluid-build' in '${name}'`);
+					throw new Error(
+						`Script task should not invoke 'fluid-build' in '${name}'. Did you forget to set 'script: false' in the task definition?`,
+					);
 				}
 			} else {
 				if (full.before.length !== 0 || full.after.length !== 0) {
 					throw new Error(
 						`Non-script task definition '${name}' cannot have 'before' or 'after'`,
 					);
+				} else if (full.files !== undefined) {
+					throw new Error(`Non-script task definition '${name}' cannot have 'files'`);
 				}
 			}
 
 			const currentTaskConfig = taskDefinitions[name];
 			full.dependsOn = expandDotDotDot(full.dependsOn, currentTaskConfig?.dependsOn);
 			full.before = expandDotDotDot(full.before, currentTaskConfig?.before);
 			full.after = expandDotDotDot(full.after, currentTaskConfig?.after);
+			const currentFiles = currentTaskConfig?.files;
+			if (full.files === undefined) {
+				full.files = currentTaskConfig?.files;
+			} else {
+				full.files.inputGlobs = expandDotDotDot(
+					full.files.inputGlobs,
+					currentFiles?.inputGlobs,
+				);
+				full.files.outputGlobs = expandDotDotDot(
+					full.files.outputGlobs,
+					currentFiles?.outputGlobs,
+				);
+			}
 			taskDefinitions[name] = full;
 		}
 	}
@@ -397,6 +497,7 @@ export function getTaskDefinitions(
 					children: directlyCalledScripts,
 					after: [],
 					script: true,
+					files: undefined,
 				};
 			} else {
 				// Confirm `children` is not specified in the manual task specifications

build-tools/packages/build-tools/src/fluidBuild/tasks/leaf/declarativeTask.ts

@@ -10,10 +10,9 @@ import {
 	type GitIgnoreSetting,
 	gitignoreDefaultValue,
 } from "../../fluidBuildConfig";
-import type { TaskHandlerFunction } from "../taskHandlers";
-import { LeafTask, LeafWithGlobInputOutputDoneFileTask } from "./leafTask";
+import { LeafWithGlobInputOutputDoneFileTask } from "./leafTask";
 
-class DeclarativeTaskHandler extends LeafWithGlobInputOutputDoneFileTask {
+export class DeclarativeLeafTask extends LeafWithGlobInputOutputDoneFileTask {
 	constructor(
 		node: BuildPackage,
 		command: string,
@@ -35,31 +34,15 @@ class DeclarativeTaskHandler extends LeafWithGlobInputOutputDoneFileTask {
 		return this.taskDefinition.gitignore ?? gitignoreDefaultValue;
 	}
 
-	protected async getInputGlobs(): Promise<string[]> {
+	protected override get includeLockFiles(): boolean {
+		return this.taskDefinition.includeLockFiles ?? super.includeLockFiles;
+	}
+
+	protected async getInputGlobs(): Promise<readonly string[]> {
 		return this.taskDefinition.inputGlobs;
 	}
 
-	protected async getOutputGlobs(): Promise<string[]> {
+	protected async getOutputGlobs(): Promise<readonly string[]> {
 		return this.taskDefinition.outputGlobs;
 	}
 }
-
-/**
- * Generates a task handler for a declarative task dynamically.
- *
- * @param taskDefinition - The declarative task definition.
- * @returns a function that can be used to instantiate a LeafTask to handle a task.
- */
-export function createDeclarativeTaskHandler(
-	taskDefinition: DeclarativeTask,
-): TaskHandlerFunction {
-	const handler: TaskHandlerFunction = (
-		node: BuildPackage,
-		command: string,
-		context: BuildContext,
-		taskName?: string,
-	): LeafTask => {
-		return new DeclarativeTaskHandler(node, command, context, taskName, taskDefinition);
-	};
-	return handler;
-}