Improve fsImporter performance by adding a resolve cache

Author: danez

.changeset/twelve-trainers-tap.md

@@ -0,0 +1,9 @@
+---
+'react-docgen': major
+---
+
+Improve performance of file system importer.
+
+The file system importer now also caches resolving of files in addition to parsing files.
+If the importer is used in an environment where files do change at runtime (like a watch
+command) then the caches will need to be cleared on every file change.

packages/react-docgen/src/importer/makeFsImporter.ts

@@ -9,30 +9,36 @@ import type { Importer, ImportPath } from './index.js';
 import type FileState from '../FileState.js';
 import { resolveObjectPatternPropertyToValue } from '../utils/index.js';
 
+// These extensions are sorted by priority
+// resolve() will check for files in the order these extensions are sorted
 const RESOLVE_EXTENSIONS = [
   '.js',
-  '.jsx',
-  '.cjs',
-  '.mjs',
   '.ts',
   '.tsx',
+  '.mjs',
+  '.cjs',
   '.mts',
   '.cts',
+  '.jsx',
 ];
 
 function defaultLookupModule(filename: string, basedir: string): string {
+  const resolveOptions = {
+    basedir,
+    extensions: RESOLVE_EXTENSIONS,
+    // we do not need to check core modules as we cannot import them anyway
+    includeCoreModules: false,
+  };
+
   try {
-    return resolve.sync(filename, {
-      basedir,
-      extensions: RESOLVE_EXTENSIONS,
-    });
+    return resolve.sync(filename, resolveOptions);
   } catch (error) {
     const ext = extname(filename);
     let newFilename: string;
 
     // if we try to import a JavaScript file it might be that we are actually pointing to
     // a TypeScript file. This can happen in ES modules as TypeScript requires to import other
-    // TypeScript files with JavaScript extensions
+    // TypeScript files with .js extensions
     // https://www.typescriptlang.org/docs/handbook/esm-node.html#type-in-packagejson-and-new-extensions
     switch (ext) {
       case '.js':
@@ -49,8 +55,9 @@ function defaultLookupModule(filename: string, basedir: string): string {
     }
 
     return resolve.sync(newFilename, {
-      basedir,
-      extensions: RESOLVE_EXTENSIONS,
+      ...resolveOptions,
+      // we already know that there is an extension at this point, so no need to check other extensions
+      extensions: [],
     });
   }
 }
@@ -62,13 +69,23 @@ interface TraverseState {
   resultPath?: NodePath | null;
 }
 
+interface FsImporterCache {
+  parseCache: Map<string, FileState>;
+  resolveCache: Map<string, string | null>;
+}
+
 // Factory for the resolveImports importer
+// If this resolver is used in an environment where the source files change (e.g. watch)
+// then the cache needs to be cleared on file changes.
 export default function makeFsImporter(
   lookupModule: (
     filename: string,
     basedir: string,
   ) => string = defaultLookupModule,
-  cache: Map<string, FileState> = new Map(),
+  { parseCache, resolveCache }: FsImporterCache = {
+    parseCache: new Map(),
+    resolveCache: new Map(),
+  },
 ): Importer {
   function resolveImportedValue(
     path: ImportPath,
@@ -87,36 +104,48 @@ export default function makeFsImporter(
 
     // Resolve the imported module using the Node resolver
     const basedir = dirname(filename);
-    let resolvedSource: string | undefined;
+    const resolveCacheKey = `${basedir}|${source}`;
+    let resolvedSource = resolveCache.get(resolveCacheKey);
+
+    // We haven't found it before, so no need to look again
+    if (resolvedSource === null) {
+      return null;
+    }
+
+    // First time we try to resolve this file
+    if (resolvedSource === undefined) {
+      try {
+        resolvedSource = lookupModule(source, basedir);
+      } catch (error) {
+        const { code } = error as NodeJS.ErrnoException;
 
-    try {
-      resolvedSource = lookupModule(source, basedir);
-    } catch (error) {
-      const { code } = error as NodeJS.ErrnoException;
+        if (code === 'MODULE_NOT_FOUND' || code === 'INVALID_PACKAGE_MAIN') {
+          resolveCache.set(resolveCacheKey, null);
 
-      if (code === 'MODULE_NOT_FOUND' || code === 'INVALID_PACKAGE_MAIN') {
-        return null;
+          return null;
+        }
+
+        throw error;
       }
 
-      throw error;
+      resolveCache.set(resolveCacheKey, resolvedSource);
     }
-
     // Prevent recursive imports
     if (seen.has(resolvedSource)) {
       return null;
     }
 
     seen.add(resolvedSource);
 
-    let nextFile = cache.get(resolvedSource);
+    let nextFile = parseCache.get(resolvedSource);
 
     if (!nextFile) {
       // Read and parse the code
       const src = fs.readFileSync(resolvedSource, 'utf8');
 
       nextFile = file.parse(src, resolvedSource);
 
-      cache.set(resolvedSource, nextFile);
+      parseCache.set(resolvedSource, nextFile);
     }
 
     return findExportedValue(nextFile, name, seen);