From 6f5f09d8b48474f6b38b1ff768b6199cd30817b2 Mon Sep 17 00:00:00 2001 From: David Michon Date: Thu, 21 Nov 2024 14:39:48 -0800 Subject: [PATCH] [heft] Add options to only resolve symlinks that are within node_modules (#5011) * [node-core-library] Add RealNodeModulePathResolver * [heft-typescript] Add onlyResolveSymlinksInNodeModules option * [heft-jest] Add node module symlink resolver * Test resolver * Use features in local-node-rig * Fix duplicate types * Disable CI fail-fast * Pay extra lstat * Fix unit tests --------- Co-authored-by: David Michon --- .github/workflows/ci.yml | 1 + .../config/jest.config.json | 3 +- .../config/typescript.json | 4 +- .../fast-realpath_2024-11-20-05-21.json | 10 + .../fast-realpath_2024-11-20-04-20.json | 10 + .../fast-realpath_2024-11-20-19-57.json | 10 + .../rush/nonbrowser-approved-packages.json | 4 + .../reviews/api/heft-typescript-plugin.api.md | 1 + common/reviews/api/node-core-library.api.md | 22 +- .../reviews/api/real-node-module-path.api.md | 17 ++ .../heft-jest-plugin/src/JestRealPathPatch.ts | 36 +++ .../jest-node-modules-symlink-resolver.ts | 7 + .../src/TypeScriptBuilder.ts | 70 ++++- .../src/TypeScriptPlugin.ts | 8 + .../internalTypings/TypeScriptInternals.ts | 16 + .../src/schemas/typescript.schema.json | 5 + .../src/RealNodeModulePath.ts | 152 ++++++++++ libraries/node-core-library/src/index.ts | 1 + .../src/test/RealNodeModulePath.test.ts | 286 ++++++++++++++++++ .../profiles/default/config/jest.config.json | 4 +- .../profiles/default/config/typescript.json | 4 +- 21 files changed, 649 insertions(+), 22 deletions(-) create mode 100644 common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json create mode 100644 common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json create mode 100644 common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json create mode 100644 common/reviews/api/real-node-module-path.api.md create mode 100644 heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts create mode 100644 heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts create mode 100644 libraries/node-core-library/src/RealNodeModulePath.ts create mode 100644 libraries/node-core-library/src/test/RealNodeModulePath.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab0d9ab1111..0459833a46c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: jobs: build: strategy: + fail-fast: false matrix: include: - NodeVersion: 18.18.x diff --git a/build-tests/heft-webpack5-everything-test/config/jest.config.json b/build-tests/heft-webpack5-everything-test/config/jest.config.json index 331b9187503..f22bb14d6d1 100644 --- a/build-tests/heft-webpack5-everything-test/config/jest.config.json +++ b/build-tests/heft-webpack5-everything-test/config/jest.config.json @@ -7,5 +7,6 @@ "coverageReporters": ["cobertura", "html"], // Use v8 coverage provider to avoid Babel - "coverageProvider": "v8" + "coverageProvider": "v8", + "resolver": "@rushstack/heft-jest-plugin/lib/exports/jest-node-modules-symlink-resolver" } diff --git a/build-tests/heft-webpack5-everything-test/config/typescript.json b/build-tests/heft-webpack5-everything-test/config/typescript.json index 29f5117a1b7..86a32ee3552 100644 --- a/build-tests/heft-webpack5-everything-test/config/typescript.json +++ b/build-tests/heft-webpack5-everything-test/config/typescript.json @@ -49,5 +49,7 @@ // "excludeGlobs": [ // "some/path/*.css" // ] - } + }, + + "onlyResolveSymlinksInNodeModules": true } diff --git a/common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json b/common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json new file mode 100644 index 00000000000..d9a6df9ecdf --- /dev/null +++ b/common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/heft-jest-plugin", + "comment": "Add a custom resolver that only resolves symlinks that are within node_modules.", + "type": "minor" + } + ], + "packageName": "@rushstack/heft-jest-plugin" +} \ No newline at end of file diff --git a/common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json b/common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json new file mode 100644 index 00000000000..15ad6be918f --- /dev/null +++ b/common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/heft-typescript-plugin", + "comment": "Add \"onlyResolveSymlinksInNodeModules\" option to improve performance for typical repository layouts.", + "type": "minor" + } + ], + "packageName": "@rushstack/heft-typescript-plugin" +} \ No newline at end of file diff --git a/common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json b/common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json new file mode 100644 index 00000000000..243969f8d18 --- /dev/null +++ b/common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Add `RealNodeModulePathResolver` class to get equivalent behavior to `realpath` with fewer system calls (and therefore higher performance) in the typical scenario where the only symlinks in the repository are inside of `node_modules` folders and are links to package folders.", + "type": "minor" + } + ], + "packageName": "@rushstack/node-core-library" +} \ No newline at end of file diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 6aa58323b73..95b79496bec 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -230,6 +230,10 @@ "name": "@rushstack/package-extractor", "allowedCategories": [ "libraries", "vscode-extensions" ] }, + { + "name": "@rushstack/real-node-module-path", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/rig-package", "allowedCategories": [ "libraries" ] diff --git a/common/reviews/api/heft-typescript-plugin.api.md b/common/reviews/api/heft-typescript-plugin.api.md index 50b95f9b678..c66a92da0ed 100644 --- a/common/reviews/api/heft-typescript-plugin.api.md +++ b/common/reviews/api/heft-typescript-plugin.api.md @@ -55,6 +55,7 @@ export interface ITypeScriptConfigurationJson { buildProjectReferences?: boolean; emitCjsExtensionForCommonJS?: boolean | undefined; emitMjsExtensionForESModule?: boolean | undefined; + onlyResolveSymlinksInNodeModules?: boolean; // (undocumented) project?: string; staticAssetsToCopy?: IStaticAssetsCopyConfiguration; diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index a11e5a042da..08cbafde6d0 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -7,7 +7,8 @@ /// import * as child_process from 'child_process'; -import * as fs from 'fs'; +import * as nodeFs from 'fs'; +import * as nodePath from 'path'; // @public export enum AlreadyExistsBehavior { @@ -213,7 +214,7 @@ export type FileSystemCopyFilesAsyncFilter = (sourcePath: string, destinationPat export type FileSystemCopyFilesFilter = (sourcePath: string, destinationPath: string) => boolean; // @public -export type FileSystemStats = fs.Stats; +export type FileSystemStats = nodeFs.Stats; // @public export class FileWriter { @@ -231,7 +232,7 @@ export const FolderConstants: { }; // @public -export type FolderItem = fs.Dirent; +export type FolderItem = nodeFs.Dirent; // @public export interface IAsyncParallelismOptions { @@ -605,6 +606,14 @@ export interface IReadLinesFromIterableOptions { ignoreEmptyLines?: boolean; } +// @public +export interface IRealNodeModulePathResolverOptions { + // (undocumented) + fs: Pick; + // (undocumented) + path: Pick; +} + // @public (undocumented) export interface IRunWithRetriesOptions { // (undocumented) @@ -834,6 +843,13 @@ export class ProtectableMap { get size(): number; } +// @public +export class RealNodeModulePathResolver { + constructor(options?: IRealNodeModulePathResolverOptions); + clearCache(): void; + readonly realNodeModulePath: (input: string) => string; +} + // @public export class Sort { static compareByValue(x: any, y: any): number; diff --git a/common/reviews/api/real-node-module-path.api.md b/common/reviews/api/real-node-module-path.api.md new file mode 100644 index 00000000000..751844943b5 --- /dev/null +++ b/common/reviews/api/real-node-module-path.api.md @@ -0,0 +1,17 @@ +## API Report File for "@rushstack/real-node-module-path" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +/// + +// @public +export function clearCache(): void; + +// @public +export const realNodeModulePath: (input: string) => string; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts b/heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts new file mode 100644 index 00000000000..4f2eb44079e --- /dev/null +++ b/heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'node:path'; +import { RealNodeModulePathResolver } from '@rushstack/node-core-library/lib/RealNodeModulePath'; + +const jestResolvePackageFolder: string = path.dirname(require.resolve('jest-resolve/package.json')); +const jestResolveFileWalkersPath: string = path.resolve(jestResolvePackageFolder, './build/fileWalkers.js'); + +const jestUtilPackageFolder: string = path.dirname( + require.resolve('jest-util/package.json', { paths: [jestResolvePackageFolder] }) +); +const jestUtilTryRealpathPath: string = path.resolve(jestUtilPackageFolder, './build/tryRealpath.js'); + +const { realNodeModulePath }: RealNodeModulePathResolver = new RealNodeModulePathResolver(); + +const fileWalkersModule: { + realpathSync: (filePath: string) => string; +} = require(jestResolveFileWalkersPath); +fileWalkersModule.realpathSync = realNodeModulePath; + +const tryRealpathModule: { + default: (filePath: string) => string; +} = require(jestUtilTryRealpathPath); +tryRealpathModule.default = (input: string): string => { + try { + return realNodeModulePath(input); + } catch (error) { + // Not using the helper from FileSystem here because this code loads in every Jest worker process + // and FileSystem has a lot of extra dependencies + if (error.code !== 'ENOENT' && error.code !== 'EISDIR') { + throw error; + } + } + return input; +}; diff --git a/heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts b/heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts new file mode 100644 index 00000000000..5c4f9fb6e70 --- /dev/null +++ b/heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import '../JestRealPathPatch'; +// Using this syntax because HeftJestResolver uses `export =` syntax. +import resolver = require('../HeftJestResolver'); +export = resolver; diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts index 4291e24cbcc..e8bad19fec6 100644 --- a/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts +++ b/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts @@ -7,14 +7,21 @@ import { Worker } from 'worker_threads'; import * as semver from 'semver'; import type * as TTypescript from 'typescript'; -import { JsonFile, type IPackageJson, Path, FileError } from '@rushstack/node-core-library'; +import { + JsonFile, + type IPackageJson, + Path, + FileError, + RealNodeModulePathResolver +} from '@rushstack/node-core-library'; import type { ITerminal } from '@rushstack/terminal'; import type { IScopedLogger } from '@rushstack/heft'; import type { ExtendedBuilderProgram, ExtendedTypeScript, - IExtendedSolutionBuilder + IExtendedSolutionBuilder, + ITypeScriptNodeSystem } from './internalTypings/TypeScriptInternals'; import type { ITypeScriptConfigurationJson } from './TypeScriptPlugin'; import type { PerformanceMeasurer } from './Performance'; @@ -314,14 +321,41 @@ export class TypeScriptBuilder { return timeout; }; + let realpath: typeof ts.sys.realpath = ts.sys.realpath; + if (this._configuration.onlyResolveSymlinksInNodeModules) { + const resolver: RealNodeModulePathResolver = new RealNodeModulePathResolver(); + realpath = resolver.realNodeModulePath; + } + + const getCurrentDirectory: () => string = () => this._configuration.buildFolderPath; + // Need to also update watchFile and watchDirectory - const system: TTypescript.System = { + const system: ITypeScriptNodeSystem = { ...ts.sys, - getCurrentDirectory: () => this._configuration.buildFolderPath, + realpath, + getCurrentDirectory, clearTimeout, setTimeout }; + if (realpath && system.getAccessibleFileSystemEntries) { + const { getAccessibleFileSystemEntries } = system; + system.readDirectory = (folderPath, extensions, exclude, include, depth): string[] => { + return ts.matchFiles( + folderPath, + extensions, + exclude, + include, + ts.sys.useCaseSensitiveFileNames, + getCurrentDirectory(), + depth, + getAccessibleFileSystemEntries, + realpath, + ts.sys.directoryExists + ); + }; + } + this._tool = { ts, system, @@ -376,7 +410,7 @@ export class TypeScriptBuilder { if (!tool.solutionBuilder && !tool.watchProgram) { //#region CONFIGURE const { duration: configureDurationMs, tsconfig } = measureTsPerformance('Configure', () => { - const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts); + const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool); this._validateTsconfig(ts, _tsconfig); return { @@ -430,7 +464,7 @@ export class TypeScriptBuilder { tsconfig, compilerHost } = measureTsPerformance('Configure', () => { - const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts); + const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool); this._validateTsconfig(ts, _tsconfig); const _compilerHost: TTypescript.CompilerHost = this._buildIncrementalCompilerHost(tool, _tsconfig); @@ -557,7 +591,7 @@ export class TypeScriptBuilder { if (!tool.solutionBuilder) { //#region CONFIGURE const { duration: configureDurationMs, solutionBuilderHost } = measureSync('Configure', () => { - const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts); + const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool); this._validateTsconfig(ts, _tsconfig); const _solutionBuilderHost: TSolutionHost = this._buildSolutionBuilderHost(tool); @@ -922,19 +956,21 @@ export class TypeScriptBuilder { return `${outFolderName}:${jsExtensionOverride || '.js'}`; } - private _loadTsconfig(ts: ExtendedTypeScript): TTypescript.ParsedCommandLine { + private _loadTsconfig(tool: ITypeScriptTool): TTypescript.ParsedCommandLine { + const { ts, system } = tool; const parsedConfigFile: ReturnType = ts.readConfigFile( this._configuration.tsconfigPath, - ts.sys.readFile + system.readFile ); const currentFolder: string = path.dirname(this._configuration.tsconfigPath); const tsconfig: TTypescript.ParsedCommandLine = ts.parseJsonConfigFileContent( parsedConfigFile.config, { - fileExists: ts.sys.fileExists, - readFile: ts.sys.readFile, - readDirectory: ts.sys.readDirectory, + fileExists: system.fileExists, + readFile: system.readFile, + readDirectory: system.readDirectory, + realpath: system.realpath, useCaseSensitiveFileNames: true }, currentFolder, @@ -1054,11 +1090,11 @@ export class TypeScriptBuilder { // Do nothing }; - const { ts } = tool; + const { ts, system } = tool; const solutionBuilderHost: TTypescript.SolutionBuilderHost = ts.createSolutionBuilderHost( - ts.sys, + system, this._getCreateBuilderProgram(ts), tool.reportDiagnostic, reportSolutionBuilderStatus, @@ -1090,7 +1126,11 @@ export class TypeScriptBuilder { if (tsconfig.options.incremental) { compilerHost = ts.createIncrementalCompilerHost(tsconfig.options, system); } else { - compilerHost = ts.createCompilerHost(tsconfig.options, undefined, system); + compilerHost = (ts.createCompilerHostWorker ?? ts.createCompilerHost)( + tsconfig.options, + undefined, + system + ); } this._changeCompilerHostToUseCache(compilerHost, tool); diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts index 3f179429062..ef174b75e4d 100644 --- a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts +++ b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts @@ -78,6 +78,12 @@ export interface ITypeScriptConfigurationJson { */ useTranspilerWorker?: boolean; + /** + * If true, the TypeScript compiler will only resolve symlinks to their targets if the links are in a node_modules folder. + * This significantly reduces file system operations in typical usage. + */ + onlyResolveSymlinksInNodeModules?: boolean; + /* * Specifies the tsconfig.json file that will be used for compilation. Equivalent to the "project" argument for the 'tsc' and 'tslint' command line tools. * @@ -365,6 +371,8 @@ export default class TypeScriptPlugin implements IHeftTaskPlugin { useTranspilerWorker: typeScriptConfigurationJson?.useTranspilerWorker, + onlyResolveSymlinksInNodeModules: typeScriptConfigurationJson?.onlyResolveSymlinksInNodeModules, + tsconfigPath: getTsconfigFilePath(heftConfiguration, typeScriptConfigurationJson), additionalModuleKindsToEmit: typeScriptConfigurationJson?.additionalModuleKindsToEmit, emitCjsExtensionForCommonJS: !!typeScriptConfigurationJson?.emitCjsExtensionForCommonJS, diff --git a/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts b/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts index c330ac96769..2b002a00efe 100644 --- a/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts +++ b/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts @@ -9,6 +9,16 @@ export interface IExtendedSolutionBuilder invalidateProject(configFilePath: string, mode: 0 | 1 | 2): void; } +export interface ITypeScriptNodeSystem extends TTypescript.System { + /** + * https://github.com/microsoft/TypeScript/blob/d85767abfd83880cea17cea70f9913e9c4496dcc/src/compiler/sys.ts#L1438 + */ + getAccessibleFileSystemEntries?: (folderPath: string) => { + files: string[]; + directories: string[]; + }; +} + export interface IExtendedTypeScript { /** * https://github.com/microsoft/TypeScript/blob/5f597e69b2e3b48d788cb548df40bcb703c8adb1/src/compiler/performance.ts#L3 @@ -59,6 +69,12 @@ export interface IExtendedTypeScript { system?: TTypescript.System ): TTypescript.CompilerHost; + createCompilerHostWorker( + options: TTypescript.CompilerOptions, + setParentNodes?: boolean, + system?: TTypescript.System + ): TTypescript.CompilerHost; + /** * https://github.com/microsoft/TypeScript/blob/782c09d783e006a697b4ba6d1e7ec2f718ce8393/src/compiler/utilities.ts#L6540 */ diff --git a/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json b/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json index dd2e15d5bad..ff66372aa51 100644 --- a/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json +++ b/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json @@ -37,6 +37,11 @@ } }, + "onlyResolveSymlinksInNodeModules": { + "description": "If true, the TypeScript compiler will only resolve symlinks to their targets if the links are in a node_modules folder. This significantly reduces file system operations in typical usage.", + "type": "boolean" + }, + "emitCjsExtensionForCommonJS": { "description": "If true, emit CommonJS module output to the folder specified in the tsconfig \"outDir\" compiler option with the .cjs extension alongside (or instead of, if TSConfig specifies CommonJS) the default compilation output.", "type": "boolean" diff --git a/libraries/node-core-library/src/RealNodeModulePath.ts b/libraries/node-core-library/src/RealNodeModulePath.ts new file mode 100644 index 00000000000..5286430fde8 --- /dev/null +++ b/libraries/node-core-library/src/RealNodeModulePath.ts @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as nodeFs from 'fs'; +import * as nodePath from 'path'; + +/** + * Arguments used to create a function that resolves symlinked node_modules in a path + * @public + */ +export interface IRealNodeModulePathResolverOptions { + fs: Pick; + path: Pick; +} + +/** + * This class encapsulates a caching resolver for symlinks in node_modules directories. + * It assumes that the only symlinks that exist in input paths are those that correspond to + * npm packages. + * + * @remarks + * In a repository with a symlinked node_modules installation, some symbolic links need to be mapped for + * node module resolution to produce correct results. However, calling `fs.realpathSync.native` on every path, + * as is commonly done by most resolvers, involves an enormous number of file system operations (for reference, + * each invocation of `fs.realpathSync.native` involves a series of `fs.readlinkSync` calls, up to one for each + * path segment in the input). + * + * @public + */ +export class RealNodeModulePathResolver { + /** + * Similar in function to `fs.realpathSync.native`, but assumes the only symlinks present are npm packages. + * + * @param input - A path to a file or directory, where the path separator is `${require('node:path').sep}` + * @returns The real path to the input, resolving the node_modules symlinks in the path + * @public + */ + public readonly realNodeModulePath: (input: string) => string; + + private readonly _cache: Map; + private readonly _fs: IRealNodeModulePathResolverOptions['fs']; + + public constructor( + options: IRealNodeModulePathResolverOptions = { + fs: nodeFs, + path: nodePath + } + ) { + const cache: Map = (this._cache = new Map()); + const { path, fs } = options; + const { sep: pathSeparator } = path; + this._fs = fs; + + const nodeModulesToken: string = `${pathSeparator}node_modules${pathSeparator}`; + + const tryReadLink: (link: string) => string | undefined = this._tryReadLink.bind(this); + + function realNodeModulePathInternal(input: string): string { + // Find the last node_modules path segment + const nodeModulesIndex: number = input.lastIndexOf(nodeModulesToken); + if (nodeModulesIndex < 0) { + // No node_modules in path, so we assume it is already the real path + return input; + } + + // First assume that the next path segment after node_modules is a symlink + let linkStart: number = nodeModulesIndex + nodeModulesToken.length - 1; + let linkEnd: number = input.indexOf(pathSeparator, linkStart + 1); + // If the path segment starts with a '@', then it is a scoped package + const isScoped: boolean = input.charAt(linkStart + 1) === '@'; + if (isScoped) { + // For a scoped package, the scope is an ordinary directory, so we need to find the next path segment + if (linkEnd < 0) { + // Symlink missing, so see if anything before the last node_modules needs resolving, + // and preserve the rest of the path + return `${realNodeModulePathInternal(input.slice(0, nodeModulesIndex))}${input.slice(nodeModulesIndex)}`; + } + + linkStart = linkEnd; + linkEnd = input.indexOf(pathSeparator, linkStart + 1); + } + + // No trailing separator, so the link is the last path segment + if (linkEnd < 0) { + linkEnd = input.length; + } + + const linkCandidate: string = input.slice(0, linkEnd); + // Check if the link is a symlink + const linkTarget: string | undefined = tryReadLink(linkCandidate); + if (linkTarget && path.isAbsolute(linkTarget)) { + // Absolute path, combine the link target with any remaining path segments + // Cache the resolution to avoid the readlink call in subsequent calls + cache.set(linkCandidate, linkTarget); + cache.set(linkTarget, linkTarget); + return `${linkTarget}${input.slice(linkEnd)}`; + } + + // Relative path or does not exist + // Either way, the path before the last node_modules could itself be in a node_modules folder + // So resolve the base path to find out what paths are relative to + const realpathBeforeNodeModules: string = realNodeModulePathInternal(input.slice(0, nodeModulesIndex)); + if (linkTarget) { + // Relative path in symbolic link. Should be resolved relative to real path of base path. + const resolvedTarget: string = path.resolve( + `${realpathBeforeNodeModules}${input.slice(nodeModulesIndex, linkStart)}`, + linkTarget + ); + // Cache the result of the combined resolution to avoid the readlink call in subsequent calls + cache.set(linkCandidate, resolvedTarget); + cache.set(resolvedTarget, resolvedTarget); + return `${resolvedTarget}${input.slice(linkEnd)}`; + } + + // No symlink, so just return the real path before the last node_modules combined with the + // subsequent path segments + return `${realpathBeforeNodeModules}${input.slice(nodeModulesIndex)}`; + } + + this.realNodeModulePath = (input: string) => { + return realNodeModulePathInternal(path.normalize(input)); + }; + } + + /** + * Clears the cache of resolved symlinks. + * @public + */ + public clearCache(): void { + this._cache.clear(); + } + + /** + * Tries to read a symbolic link at the specified path. + * If the input is not a symbolic link, returns undefined. + * @param link - The link to try to read + * @returns The target of the symbolic link, or undefined if the input is not a symbolic link + */ + private _tryReadLink(link: string): string | undefined { + const cached: string | undefined = this._cache.get(link); + if (cached) { + return cached; + } + + // On Windows, calling `readlink` on a directory throws an EUNKOWN, not EINVAL, so just pay the cost + // of an lstat call. + const stat: nodeFs.Stats | undefined = this._fs.lstatSync(link); + if (stat.isSymbolicLink()) { + return this._fs.readlinkSync(link, 'utf8'); + } + } +} diff --git a/libraries/node-core-library/src/index.ts b/libraries/node-core-library/src/index.ts index 54fb6509b7a..531805cfe6b 100644 --- a/libraries/node-core-library/src/index.ts +++ b/libraries/node-core-library/src/index.ts @@ -93,6 +93,7 @@ export { type IPathFormatFileLocationOptions, type IPathFormatConciselyOptions } from './Path'; +export { RealNodeModulePathResolver, type IRealNodeModulePathResolverOptions } from './RealNodeModulePath'; export { Encoding, Text, NewlineKind, type IReadLinesFromIterableOptions } from './Text'; export { Sort } from './Sort'; export { diff --git a/libraries/node-core-library/src/test/RealNodeModulePath.test.ts b/libraries/node-core-library/src/test/RealNodeModulePath.test.ts new file mode 100644 index 00000000000..ff906001678 --- /dev/null +++ b/libraries/node-core-library/src/test/RealNodeModulePath.test.ts @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type * as fs from 'fs'; +import * as path from 'path'; + +import { type IRealNodeModulePathResolverOptions, RealNodeModulePathResolver } from '../RealNodeModulePath'; + +const mocklstatSync: jest.Mock, Parameters> = jest.fn(); +const lstatSync: typeof fs.lstatSync = mocklstatSync as unknown as typeof fs.lstatSync; +const mockReadlinkSync: jest.Mock< + ReturnType, + Parameters +> = jest.fn(); +const readlinkSync: typeof fs.readlinkSync = mockReadlinkSync as unknown as typeof fs.readlinkSync; + +const mockFs: IRealNodeModulePathResolverOptions['fs'] = { + lstatSync, + readlinkSync +}; + +describe('realNodeModulePath', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('POSIX paths', () => { + const resolver: RealNodeModulePathResolver = new RealNodeModulePathResolver({ + fs: mockFs, + path: path.posix + }); + const { realNodeModulePath } = resolver; + + beforeEach(() => { + resolver.clearCache(); + }); + + it('should return the input path if it does not contain node_modules', () => { + for (const input of ['/foo/bar', '/', 'ab', '../foo/bar/baz']) { + expect(realNodeModulePath(input)).toBe(input); + + expect(mocklstatSync).not.toHaveBeenCalled(); + expect(mockReadlinkSync).not.toHaveBeenCalled(); + } + }); + + it('should return the input path if it is not a symbolic link', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => false } as unknown as fs.Stats); + + expect(realNodeModulePath('/foo/node_modules/foo')).toBe('/foo/node_modules/foo'); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/node_modules/foo'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledTimes(0); + }); + + it('Should handle absolute link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('/link/target'); + + expect(realNodeModulePath('/foo/node_modules/link')).toBe('/link/target'); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Caches resolved symlinks', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('/link/target'); + + expect(realNodeModulePath('/foo/node_modules/link')).toBe('/link/target'); + expect(realNodeModulePath('/foo/node_modules/link/bar')).toBe('/link/target/bar'); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should stop after a single absolute link target', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('/link/target'); + + expect(realNodeModulePath('/node_modules/foo/node_modules/link')).toBe('/link/target'); + + expect(mocklstatSync).toHaveBeenCalledWith('/node_modules/foo/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('/node_modules/foo/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should handle relative link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('../../link/target'); + + expect(realNodeModulePath('/foo/node_modules/link')).toBe('/link/target'); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should recursively handle relative link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('../../link'); + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('/other/root/bar'); + + expect(realNodeModulePath('/foo/1/2/3/node_modules/bar/node_modules/link/4/5/6')).toBe( + '/other/root/link/4/5/6' + ); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar'); + expect(mocklstatSync).toHaveBeenCalledTimes(2); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(2); + }); + + it('Caches multi-layer resolution', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('../../link'); + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('/other/root/bar'); + + expect(realNodeModulePath('/foo/1/2/3/node_modules/bar/node_modules/link/4/5/6')).toBe( + '/other/root/link/4/5/6' + ); + expect(realNodeModulePath('/foo/1/2/3/node_modules/bar/node_modules/link/a/b')).toBe( + '/other/root/link/a/b' + ); + expect(realNodeModulePath('/foo/1/2/3/node_modules/bar/a/b')).toBe('/other/root/bar/a/b'); + + expect(mocklstatSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar/node_modules/link'); + expect(mocklstatSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar'); + expect(mocklstatSync).toHaveBeenCalledTimes(2); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar/node_modules/link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledWith('/foo/1/2/3/node_modules/bar', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(2); + }); + }); + + describe('Windows paths', () => { + const resolver: RealNodeModulePathResolver = new RealNodeModulePathResolver({ + fs: mockFs, + path: path.win32 + }); + const { realNodeModulePath } = resolver; + + beforeEach(() => { + resolver.clearCache(); + }); + + it('should return the input path if it does not contain node_modules', () => { + for (const input of ['C:\\foo\\bar', 'C:\\', 'ab', '..\\foo\\bar\\baz']) { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + + expect(realNodeModulePath(input)).toBe(input); + + expect(mocklstatSync).not.toHaveBeenCalled(); + expect(mockReadlinkSync).not.toHaveBeenCalled(); + } + }); + + it('should return the normalized input path if it does not contain node_modules', () => { + for (const input of ['C:/foo/bar', 'C:/', 'ab', '../foo/bar/baz']) { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + + expect(realNodeModulePath(input)).toBe(path.win32.normalize(input)); + + expect(mocklstatSync).not.toHaveBeenCalled(); + expect(mockReadlinkSync).not.toHaveBeenCalled(); + } + }); + + it('Should return the input path if the target is not a symbolic link', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => false } as unknown as fs.Stats); + + expect(realNodeModulePath('C:\\foo\\node_modules\\foo')).toBe('C:\\foo\\node_modules\\foo'); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\foo'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledTimes(0); + }); + + it('Should handle absolute link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('C:\\link\\target'); + + expect(realNodeModulePath('C:\\foo\\node_modules\\link')).toBe('C:\\link\\target'); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should normalize input', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('C:\\link\\target'); + + expect(realNodeModulePath('C:\\foo\\node_modules\\link')).toBe('C:\\link\\target'); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should stop after a single absolute link target', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('D:\\link\\target'); + + expect(realNodeModulePath('C:\\node_modules\\foo\\node_modules\\link')).toBe('D:\\link\\target'); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\node_modules\\foo\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\node_modules\\foo\\node_modules\\link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should handle relative link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('..\\..\\link\\target'); + + expect(realNodeModulePath('C:\\foo\\node_modules\\link')).toBe('C:\\link\\target'); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledTimes(1); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\foo\\node_modules\\link', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(1); + }); + + it('Should recursively handle relative link targets', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('..\\..\\link'); + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('D:\\other\\root\\bar'); + + expect(realNodeModulePath('C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link\\4\\5\\6')).toBe( + 'D:\\other\\root\\link\\4\\5\\6' + ); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar'); + expect(mocklstatSync).toHaveBeenCalledTimes(2); + expect(mockReadlinkSync).toHaveBeenCalledWith( + 'C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link', + 'utf8' + ); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(2); + }); + + it('Caches multi-layer resolution', () => { + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('..\\..\\link'); + mocklstatSync.mockReturnValueOnce({ isSymbolicLink: () => true } as unknown as fs.Stats); + mockReadlinkSync.mockReturnValueOnce('D:\\other\\root\\bar'); + + expect(realNodeModulePath('C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link\\4\\5\\6')).toBe( + 'D:\\other\\root\\link\\4\\5\\6' + ); + expect(realNodeModulePath('C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link\\a\\b')).toBe( + 'D:\\other\\root\\link\\a\\b' + ); + expect(realNodeModulePath('C:\\foo\\1\\2\\3\\node_modules\\bar\\a\\b')).toBe( + 'D:\\other\\root\\bar\\a\\b' + ); + + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link'); + expect(mocklstatSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar'); + expect(mocklstatSync).toHaveBeenCalledTimes(2); + expect(mockReadlinkSync).toHaveBeenCalledWith( + 'C:\\foo\\1\\2\\3\\node_modules\\bar\\node_modules\\link', + 'utf8' + ); + expect(mockReadlinkSync).toHaveBeenCalledWith('C:\\foo\\1\\2\\3\\node_modules\\bar', 'utf8'); + expect(mockReadlinkSync).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/rigs/local-node-rig/profiles/default/config/jest.config.json b/rigs/local-node-rig/profiles/default/config/jest.config.json index 969ec5d5fb2..1127530a185 100644 --- a/rigs/local-node-rig/profiles/default/config/jest.config.json +++ b/rigs/local-node-rig/profiles/default/config/jest.config.json @@ -20,5 +20,7 @@ "titleTemplate": "{title} ({filepath})" } ] - ] + ], + + "resolver": "@rushstack/heft-jest-plugin/lib/exports/jest-node-modules-symlink-resolver.js" } diff --git a/rigs/local-node-rig/profiles/default/config/typescript.json b/rigs/local-node-rig/profiles/default/config/typescript.json index 03fb43fc7ac..e821bdb4c18 100644 --- a/rigs/local-node-rig/profiles/default/config/typescript.json +++ b/rigs/local-node-rig/profiles/default/config/typescript.json @@ -1,5 +1,7 @@ { "$schema": "https://developer.microsoft.com/json-schemas/heft/typescript.schema.json", - "extends": "@rushstack/heft-node-rig/profiles/default/config/typescript.json" + "extends": "@rushstack/heft-node-rig/profiles/default/config/typescript.json", + + "onlyResolveSymlinksInNodeModules": true }