[rush-lib] Ignore specified files when calculating project state hash

apps/rush-lib/src/api/RushProjectConfiguration.ts

@@ -21,6 +21,18 @@ interface IRushProjectJson {
    */
   projectOutputFolderNames?: string[];
 
+  /**
+   * The incremental analyzer can skip Rush commands for projects whose input files have
+   * not changed since the last build. Normally, every Git-tracked file under the project
+   * folder is assumed to be an input. Set incrementalBuildIgnoredGlobs to ignore specific
+   * files, specified as globs relative to the project folder. The list of file globs will
+   * be interpreted the same way your .gitignore file is.
+   */
+  incrementalBuildIgnoredGlobs?: string[];
+
+  /**
+   * Additional project-specific options related to build caching.
+   */
   buildCacheOptions?: IBuildCacheOptionsJson;
 }
 
@@ -86,6 +98,9 @@ export class RushProjectConfiguration {
         projectOutputFolderNames: {
           inheritanceType: InheritanceType.append
         },
+        incrementalBuildIgnoredGlobs: {
+          inheritanceType: InheritanceType.replace
+        },
         buildCacheOptions: {
           inheritanceType: InheritanceType.custom,
           inheritanceFunction: (
@@ -121,6 +136,15 @@ export class RushProjectConfiguration {
    */
   public readonly projectOutputFolderNames?: string[];
 
+  /**
+   * The incremental analyzer can skip Rush commands for projects whose input files have
+   * not changed since the last build. Normally, every Git-tracked file under the project
+   * folder is assumed to be an input. Set incrementalBuildIgnoredGlobs to ignore specific
+   * files, specified as globs relative to the project folder. The list of file globs will
+   * be interpreted the same way your .gitignore file is.
+   */
+  public readonly incrementalBuildIgnoredGlobs?: string[];
+
   /**
    * Project-specific cache options.
    */
@@ -131,6 +155,8 @@ export class RushProjectConfiguration {
 
     this.projectOutputFolderNames = rushProjectJson.projectOutputFolderNames;
 
+    this.incrementalBuildIgnoredGlobs = rushProjectJson.incrementalBuildIgnoredGlobs;
+
     const optionsForCommandsByName: Map<string, ICacheOptionsForCommand> = new Map<
       string,
       ICacheOptionsForCommand

apps/rush-lib/src/cli/actions/WriteBuildCacheAction.ts

@@ -87,7 +87,7 @@ export class WriteBuildCacheAction extends BaseRushAction {
     });
 
     const trackedFiles: string[] = Array.from(
-      packageChangeAnalyzer.getPackageDeps(project.packageName)!.keys()
+      (await packageChangeAnalyzer.getPackageDeps(project.packageName, terminal))!.keys()
     );
     const commandLineConfigFilePath: string = path.join(
       this.rushConfiguration.commonRushConfigFolder,

apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts

@@ -191,7 +191,8 @@ export class BulkScriptAction extends BaseScriptAction {
     const projectWatcher: typeof ProjectWatcher.prototype = new ProjectWatcher({
       debounceMilliseconds: 1000,
       rushConfiguration: this.rushConfiguration,
-      projectsToWatch
+      projectsToWatch,
+      terminal
     });
 
     let isInitialPass: boolean = true;

apps/rush-lib/src/logic/PackageChangeAnalyzer.ts

@@ -4,11 +4,13 @@
 import * as path from 'path';
 import colors from 'colors/safe';
 import * as crypto from 'crypto';
+import ignore, { Ignore } from 'ignore';
 
 import { getPackageDeps, getGitHashForFiles } from '@rushstack/package-deps-hash';
-import { Path, InternalError, FileSystem } from '@rushstack/node-core-library';
+import { Path, InternalError, FileSystem, Terminal, Async } from '@rushstack/node-core-library';
 
 import { RushConfiguration } from '../api/RushConfiguration';
+import { RushProjectConfiguration } from '../api/RushProjectConfiguration';
 import { Git } from './Git';
 import { BaseProjectShrinkwrapFile } from './base/BaseProjectShrinkwrapFile';
 import { RushConfigurationProject } from '../api/RushConfigurationProject';
@@ -29,9 +31,12 @@ export class PackageChangeAnalyzer {
     this._git = new Git(this._rushConfiguration);
   }
 
-  public getPackageDeps(projectName: string): Map<string, string> | undefined {
+  public async getPackageDeps(
+    projectName: string,
+    terminal: Terminal
+  ): Promise<Map<string, string> | undefined> {
     if (this._data === null) {
-      this._data = this._getData();
+      this._data = await this._getData(terminal);
     }
 
     return this._data?.get(projectName);
@@ -47,10 +52,10 @@ export class PackageChangeAnalyzer {
    *   Git SHA is fed into the hash
    * - A hex digest of the hash is returned
    */
-  public getProjectStateHash(projectName: string): string | undefined {
+  public async getProjectStateHash(projectName: string, terminal: Terminal): Promise<string | undefined> {
     let projectState: string | undefined = this._projectStateCache.get(projectName);
     if (!projectState) {
-      const packageDeps: Map<string, string> | undefined = this.getPackageDeps(projectName);
+      const packageDeps: Map<string, string> | undefined = await this.getPackageDeps(projectName, terminal);
       if (!packageDeps) {
         return undefined;
       } else {
@@ -71,18 +76,27 @@ export class PackageChangeAnalyzer {
     return projectState;
   }
 
-  private _getData(): Map<string, Map<string, string>> | undefined {
+  private async _getData(terminal: Terminal): Promise<Map<string, Map<string, string>> | undefined> {
     const repoDeps: Map<string, string> | undefined = this._getRepoDeps();
     if (!repoDeps) {
       return undefined;
     }
 
     const projectHashDeps: Map<string, Map<string, string>> = new Map<string, Map<string, string>>();
-
-    // pre-populate the map with the projects from the config
-    for (const project of this._rushConfiguration.projects) {
-      projectHashDeps.set(project.packageName, new Map<string, string>());
-    }
+    const ignoreMatcherForProject: Map<string, Ignore> = new Map<string, Ignore>();
+
+    // Initialize maps for each project asynchronously, up to 10 projects concurrently.
+    await Async.forEachAsync(
+      this._rushConfiguration.projects,
+      async (project: RushConfigurationProject): Promise<void> => {
+        projectHashDeps.set(project.packageName, new Map<string, string>());
+        ignoreMatcherForProject.set(
+          project.packageName,
+          await this._getIgnoreMatcherForProject(project, terminal)
+        );
+      },
+      { concurrency: 10 }
+    );
 
     // Sort each project folder into its own package deps hash
     for (const [filePath, fileHash] of repoDeps) {
@@ -92,7 +106,14 @@ export class PackageChangeAnalyzer {
         | RushConfigurationProject
         | undefined = this._rushConfiguration.findProjectForPosixRelativePath(filePath);
       if (owningProject) {
-        projectHashDeps.get(owningProject.packageName)!.set(filePath, fileHash);
+        // At this point, `filePath` is guaranteed to start with `projectRelativeFolder`, so
+        // we can safely slice off the first N characters to get the file path relative to the
+        // root of the `owningProject`.
+        const relativePath: string = filePath.slice(owningProject.projectRelativeFolder.length + 1);
+        const ignoreMatcher: Ignore | undefined = ignoreMatcherForProject.get(owningProject.packageName);
+        if (!ignoreMatcher || !ignoreMatcher.ignores(relativePath)) {
+          projectHashDeps.get(owningProject.packageName)!.set(filePath, fileHash);
+        }
       }
     }
 
@@ -157,6 +178,22 @@ export class PackageChangeAnalyzer {
     return projectHashDeps;
   }
 
+  private async _getIgnoreMatcherForProject(
+    project: RushConfigurationProject,
+    terminal: Terminal
+  ): Promise<Ignore> {
+    const projectConfiguration:
+      | RushProjectConfiguration
+      | undefined = await RushProjectConfiguration.tryLoadForProjectAsync(project, undefined, terminal);
+    const ignoreMatcher: Ignore = ignore();
+
+    if (projectConfiguration && projectConfiguration.incrementalBuildIgnoredGlobs) {
+      ignoreMatcher.add(projectConfiguration.incrementalBuildIgnoredGlobs);
+    }
+
+    return ignoreMatcher;
+  }
+
   private _getRepoDeps(): Map<string, string> | undefined {
     try {
       if (this._git.isPathUnderGitWorkingTree()) {

apps/rush-lib/src/logic/ProjectWatcher.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
-import { Import, Path } from '@rushstack/node-core-library';
+import { Import, Path, Terminal } from '@rushstack/node-core-library';
 
 import { PackageChangeAnalyzer } from './PackageChangeAnalyzer';
 import { RushConfiguration } from '../api/RushConfiguration';
@@ -14,6 +14,7 @@ export interface IProjectWatcherOptions {
   debounceMilliseconds?: number;
   rushConfiguration: RushConfiguration;
   projectsToWatch: ReadonlySet<RushConfigurationProject>;
+  terminal: Terminal;
 }
 
 export interface IProjectChangeResult {
@@ -37,16 +38,18 @@ export class ProjectWatcher {
   private readonly _debounceMilliseconds: number;
   private readonly _rushConfiguration: RushConfiguration;
   private readonly _projectsToWatch: ReadonlySet<RushConfigurationProject>;
+  private readonly _terminal: Terminal;
 
   private _initialState: PackageChangeAnalyzer | undefined;
   private _previousState: PackageChangeAnalyzer | undefined;
 
   public constructor(options: IProjectWatcherOptions) {
-    const { debounceMilliseconds = 1000, rushConfiguration, projectsToWatch } = options;
+    const { debounceMilliseconds = 1000, rushConfiguration, projectsToWatch, terminal } = options;
 
     this._debounceMilliseconds = debounceMilliseconds;
     this._rushConfiguration = rushConfiguration;
     this._projectsToWatch = projectsToWatch;
+    this._terminal = terminal;
   }
 
   /**
@@ -55,7 +58,7 @@ export class ProjectWatcher {
    * If no change is currently present, watches the source tree of all selected projects for file changes.
    */
   public async waitForChange(): Promise<IProjectChangeResult> {
-    const initialChangeResult: IProjectChangeResult = this._computeChanged();
+    const initialChangeResult: IProjectChangeResult = await this._computeChanged();
     // Ensure that the new state is recorded so that we don't loop infinitely
     this._commitChanges(initialChangeResult.state);
     if (initialChangeResult.changedProjects.size) {
@@ -82,14 +85,14 @@ export class ProjectWatcher {
         let timeout: NodeJS.Timeout | undefined;
         let terminated: boolean = false;
 
-        const resolveIfChanged = (): void => {
+        const resolveIfChanged = async (): Promise<void> => {
           timeout = undefined;
           if (terminated) {
             return;
           }
 
           try {
-            const result: IProjectChangeResult = this._computeChanged();
+            const result: IProjectChangeResult = await this._computeChanged();
 
             // Need an async tick to allow for more file system events to be handled
             process.nextTick(() => {
@@ -106,6 +109,7 @@ export class ProjectWatcher {
               }
             });
           } catch (err) {
+            // eslint-disable-next-line require-atomic-updates
             terminated = true;
             reject(err);
           }
@@ -139,7 +143,7 @@ export class ProjectWatcher {
   /**
    * Determines which, if any, projects (within the selection) have new hashes for files that are not in .gitignore
    */
-  private _computeChanged(): IProjectChangeResult {
+  private async _computeChanged(): Promise<IProjectChangeResult> {
     const state: PackageChangeAnalyzer = new PackageChangeAnalyzer(this._rushConfiguration);
 
     const previousState: PackageChangeAnalyzer | undefined = this._previousState;
@@ -157,8 +161,8 @@ export class ProjectWatcher {
 
       if (
         ProjectWatcher._haveProjectDepsChanged(
-          previousState.getPackageDeps(packageName)!,
-          state.getPackageDeps(packageName)!
+          (await previousState.getPackageDeps(packageName, this._terminal))!,
+          (await state.getPackageDeps(packageName, this._terminal))!
         )
       ) {
         // May need to detect if the nature of the change will break the process, e.g. changes to package.json

apps/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

@@ -45,15 +45,15 @@ export class ProjectBuildCache {
   private readonly _cloudBuildCacheProvider: CloudBuildCacheProviderBase | undefined;
   private readonly _buildCacheEnabled: boolean;
   private readonly _projectOutputFolderNames: string[];
-  private readonly _cacheId: string | undefined;
+  private _cacheId: string | undefined;
 
-  private constructor(options: Omit<IProjectBuildCacheOptions, 'terminal'>) {
+  private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) {
     this._project = options.projectConfiguration.project;
     this._localBuildCacheProvider = options.buildCacheConfiguration.localCacheProvider;
     this._cloudBuildCacheProvider = options.buildCacheConfiguration.cloudCacheProvider;
     this._buildCacheEnabled = options.buildCacheConfiguration.buildCacheEnabled;
     this._projectOutputFolderNames = options.projectConfiguration.projectOutputFolderNames || [];
-    this._cacheId = ProjectBuildCache._getCacheId(options);
+    this._cacheId = cacheId;
   }
 
   private static _tryGetTarUtility(terminal: Terminal): TarExecutable | undefined {
@@ -64,7 +64,9 @@ export class ProjectBuildCache {
     return ProjectBuildCache._tarUtility;
   }
 
-  public static tryGetProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache | undefined {
+  public static async tryGetProjectBuildCache(
+    options: IProjectBuildCacheOptions
+  ): Promise<ProjectBuildCache | undefined> {
     const { terminal, projectConfiguration, trackedProjectFiles } = options;
     if (!trackedProjectFiles) {
       return undefined;
@@ -74,7 +76,8 @@ export class ProjectBuildCache {
       return undefined;
     }
 
-    return new ProjectBuildCache(options);
+    const cacheId: string | undefined = await ProjectBuildCache._getCacheId(options);
+    return new ProjectBuildCache(cacheId, options);
   }
 
   private static _validateProject(
@@ -418,7 +421,7 @@ export class ProjectBuildCache {
     return path.join(this._project.projectRushTempFolder, 'build-cache-tar.log');
   }
 
-  private static _getCacheId(options: Omit<IProjectBuildCacheOptions, 'terminal'>): string | undefined {
+  private static async _getCacheId(options: IProjectBuildCacheOptions): Promise<string | undefined> {
     // The project state hash is calculated in the following method:
     // - The current project's hash (see PackageChangeAnalyzer.getProjectStateHash) is
     //   calculated and appended to an array
@@ -442,8 +445,9 @@ export class ProjectBuildCache {
       for (const projectToProcess of projectsToProcess) {
         projectsThatHaveBeenProcessed.add(projectToProcess);
 
-        const projectState: string | undefined = packageChangeAnalyzer.getProjectStateHash(
-          projectToProcess.packageName
+        const projectState: string | undefined = await packageChangeAnalyzer.getProjectStateHash(
+          projectToProcess.packageName,
+          options.terminal
         );
         if (!projectState) {
           // If we hit any projects with unknown state, return unknown cache ID

apps/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts

@@ -17,15 +17,15 @@ interface ITestOptions {
 }
 
 describe('ProjectBuildCache', () => {
-  function prepareSubject(options: Partial<ITestOptions>): ProjectBuildCache | undefined {
+  async function prepareSubject(options: Partial<ITestOptions>): Promise<ProjectBuildCache | undefined> {
     const terminal: Terminal = new Terminal(new StringBufferTerminalProvider());
     const packageChangeAnalyzer = ({
       getProjectStateHash: () => {
         return 'state_hash';
       }
     } as unknown) as PackageChangeAnalyzer;
 
-    const subject: ProjectBuildCache | undefined = ProjectBuildCache.tryGetProjectBuildCache({
+    const subject: ProjectBuildCache | undefined = await ProjectBuildCache.tryGetProjectBuildCache({
       buildCacheConfiguration: ({
         buildCacheEnabled: options.hasOwnProperty('enabled') ? options.enabled : true,
         getCacheEntryId: (options: IGenerateCacheEntryIdOptions) =>
@@ -53,16 +53,16 @@ describe('ProjectBuildCache', () => {
   }
 
   describe('tryGetProjectBuildCache', () => {
-    it('returns a ProjectBuildCache with a calculated cacheId value', () => {
-      const subject: ProjectBuildCache = prepareSubject({})!;
+    it('returns a ProjectBuildCache with a calculated cacheId value', async () => {
+      const subject: ProjectBuildCache = (await prepareSubject({}))!;
       expect(subject['_cacheId']).toMatchInlineSnapshot(
         `"acme-wizard/e229f8765b7d450a8a84f711a81c21e37935d661"`
       );
     });
 
-    it('returns undefined if the tracked file list is undefined', () => {
+    it('returns undefined if the tracked file list is undefined', async () => {
       expect(
-        prepareSubject({
+        await prepareSubject({
           trackedProjectFiles: undefined
         })
       ).toBe(undefined);