Clean up require.cache handling (#68743)

Follow-up from #68535.

Since the `require.cache` handling is not specific for Webpack, we move the helper functions out of `webpack/plugins/nextjs-require-cache-hot-reloader.ts`.

In addition, when deleting an entry from the `require.cache`, while cleaning up its references in the `children` of parent modules, we now consider all entries in the `require.cache` instead of relying on a list of "origin modules". This list had entries that were not needed, most notably the server runtime bundles (the experimental runtimes were even missing), and at least one (`node-manifest-loader`) was also missing.

This has a negligible effect on performance. Handling 1200 `require.cache` entries takes 0.3ms on an M2. The approach was also previously aligned with @sokra.

Author: unstubbable

packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts

@@ -1,69 +1,12 @@
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
+import { deleteCache } from '../../../server/dev/require-cache'
 import { clearModuleContext } from '../../../server/web/sandbox'
-import { realpathSync } from '../../../lib/realpath'
 import path from 'path'
-import isError from '../../../lib/is-error'
-import { clearManifestCache } from '../../../server/load-manifest'
 
 type Compiler = webpack.Compiler
 type WebpackPluginInstance = webpack.WebpackPluginInstance
 
-const originModules = [
-  require.resolve('../../../server/require'),
-  require.resolve('../../../server/load-components'),
-  require.resolve('../../../server/next-server'),
-  require.resolve('next/dist/compiled/next-server/app-page.runtime.dev.js'),
-  require.resolve('next/dist/compiled/next-server/app-route.runtime.dev.js'),
-  require.resolve('next/dist/compiled/next-server/pages.runtime.dev.js'),
-  require.resolve('next/dist/compiled/next-server/pages-api.runtime.dev.js'),
-]
-
 const RUNTIME_NAMES = ['webpack-runtime', 'webpack-api-runtime']
-
-function deleteFromRequireCache(filePath: string) {
-  try {
-    filePath = realpathSync(filePath)
-  } catch (e) {
-    if (isError(e) && e.code !== 'ENOENT') throw e
-  }
-  const mod = require.cache[filePath]
-  if (mod) {
-    // remove the child reference from the originModules
-    for (const originModule of originModules) {
-      const parent = require.cache[originModule]
-      if (parent) {
-        const idx = parent.children.indexOf(mod)
-        if (idx >= 0) parent.children.splice(idx, 1)
-      }
-    }
-    // remove parent references from external modules
-    for (const child of mod.children) {
-      child.parent = null
-    }
-    delete require.cache[filePath]
-    return true
-  }
-  return false
-}
-
-export function deleteAppClientCache() {
-  deleteFromRequireCache(
-    require.resolve('next/dist/compiled/next-server/app-page.runtime.dev.js')
-  )
-  deleteFromRequireCache(
-    require.resolve(
-      'next/dist/compiled/next-server/app-page-experimental.runtime.dev.js'
-    )
-  )
-}
-
-export function deleteCache(filePath: string) {
-  // try to clear it from the fs cache
-  clearManifestCache(filePath)
-
-  deleteFromRequireCache(filePath)
-}
-
 const PLUGIN_NAME = 'NextJsRequireCacheHotReloader'
 
 // This plugin flushes require.cache after emitting the files. Providing 'hot reloading' of server files.

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -31,10 +31,7 @@ import { BLOCKED_PAGES } from '../../shared/lib/constants'
 import { getOverlayMiddleware } from '../../client/components/react-dev-overlay/server/middleware-turbopack'
 import { PageNotFoundError } from '../../shared/lib/utils'
 import { debounce } from '../utils'
-import {
-  deleteAppClientCache,
-  deleteCache,
-} from '../../build/webpack/plugins/nextjs-require-cache-hot-reloader'
+import { deleteAppClientCache, deleteCache } from './require-cache'
 import {
   clearAllModuleContexts,
   clearModuleContext,

packages/next/src/server/dev/require-cache.ts

@@ -0,0 +1,46 @@
+import isError from '../../lib/is-error'
+import { realpathSync } from '../../lib/realpath'
+import { clearManifestCache } from '../load-manifest'
+
+function deleteFromRequireCache(filePath: string) {
+  try {
+    filePath = realpathSync(filePath)
+  } catch (e) {
+    if (isError(e) && e.code !== 'ENOENT') throw e
+  }
+  const mod = require.cache[filePath]
+  if (mod) {
+    // remove the child reference from all parent modules
+    for (const parent of Object.values(require.cache)) {
+      if (parent?.children) {
+        const idx = parent.children.indexOf(mod)
+        if (idx >= 0) parent.children.splice(idx, 1)
+      }
+    }
+    // remove parent references from external modules
+    for (const child of mod.children) {
+      child.parent = null
+    }
+    delete require.cache[filePath]
+    return true
+  }
+  return false
+}
+
+export function deleteAppClientCache() {
+  deleteFromRequireCache(
+    require.resolve('next/dist/compiled/next-server/app-page.runtime.dev.js')
+  )
+  deleteFromRequireCache(
+    require.resolve(
+      'next/dist/compiled/next-server/app-page-experimental.runtime.dev.js'
+    )
+  )
+}
+
+export function deleteCache(filePath: string) {
+  // try to clear it from the fs cache
+  clearManifestCache(filePath)
+
+  deleteFromRequireCache(filePath)
+}

packages/next/src/server/dev/turbopack/manifest-loader.ts

@@ -27,7 +27,7 @@ import {
 import { join, posix } from 'path'
 import { readFile, writeFile } from 'fs/promises'
 import type { SetupOpts } from '../../lib/router-utils/setup-dev-bundler'
-import { deleteCache } from '../../../build/webpack/plugins/nextjs-require-cache-hot-reloader'
+import { deleteCache } from '../require-cache'
 import { writeFileAtomic } from '../../../lib/fs/write-atomic'
 import { isInterceptionRouteRewrite } from '../../../lib/generate-interception-routes-rewrites'
 import {

packages/next/src/server/lib/render-server.ts

@@ -20,13 +20,9 @@ let initializations: Record<
 > = {}
 
 let sandboxContext: undefined | typeof import('../web/sandbox/context')
-let requireCacheHotReloader:
-  | undefined
-  | typeof import('../../build/webpack/plugins/nextjs-require-cache-hot-reloader')
 
 if (process.env.NODE_ENV !== 'production') {
   sandboxContext = require('../web/sandbox/context')
-  requireCacheHotReloader = require('../../build/webpack/plugins/nextjs-require-cache-hot-reloader')
 }
 
 export function clearAllModuleContexts() {
@@ -37,14 +33,17 @@ export function clearModuleContext(target: string) {
   return sandboxContext?.clearModuleContext(target)
 }
 
-export function deleteAppClientCache() {
-  return requireCacheHotReloader?.deleteAppClientCache()
-}
-
-export function deleteCache(filePaths: string[]) {
-  for (const filePath of filePaths) {
-    requireCacheHotReloader?.deleteCache(filePath)
+export async function getServerField(
+  dir: string,
+  field: PropagateToWorkersField
+) {
+  const initialization = await initializations[dir]
+  if (!initialization) {
+    throw new Error('Invariant cant propagate server field, no app initialized')
   }
+  const { app } = initialization
+  let appField = (app as any).server
+  return appField[field]
 }
 
 export async function propagateServerField(
@@ -77,7 +76,6 @@ async function initializeImpl(opts: {
   dev: boolean
   minimalMode?: boolean
   hostname?: string
-  isNodeDebugging: boolean
   keepAliveTimeout?: number
   serverFields?: any
   server?: any
@@ -87,6 +85,7 @@ async function initializeImpl(opts: {
   _ipcKey?: string
   bundlerService: DevBundlerService | undefined
   startServerSpan: Span | undefined
+  quiet?: boolean
 }) {
   const type = process.env.__NEXT_PRIVATE_RENDER_WORKER
   if (type) {
@@ -102,7 +101,6 @@ async function initializeImpl(opts: {
     customServer: false,
     httpServer: opts.server,
     port: opts.port,
-    isNodeDebugging: opts.isNodeDebugging,
   })
   requestHandler = app.getRequestHandler()
   upgradeHandler = app.getUpgradeHandler()

packages/next/src/server/lib/router-server.ts

@@ -47,11 +47,7 @@ const isNextFont = (pathname: string | null) =>
 
 export type RenderServer = Pick<
   typeof import('./render-server'),
-  | 'initialize'
-  | 'deleteCache'
-  | 'clearModuleContext'
-  | 'deleteAppClientCache'
-  | 'propagateServerField'
+  'initialize' | 'clearModuleContext' | 'propagateServerField'
 >
 
 export interface LazyRenderServerInstance {