diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a741e858daaa..14a095351aa4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -956,6 +956,7 @@ jobs: afterBuild: | export __NEXT_EXPERIMENTAL_PPR=true # for compatibility with the existing tests export __NEXT_EXPERIMENTAL_CACHE_COMPONENTS=true + export __NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION=true export NEXT_EXTERNAL_TESTS_FILTERS="test/experimental-tests-manifest.json" node run-tests.js \ @@ -978,6 +979,7 @@ jobs: afterBuild: | export __NEXT_EXPERIMENTAL_PPR=true # for compatibility with the existing tests export __NEXT_EXPERIMENTAL_CACHE_COMPONENTS=true + export __NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION=true export NEXT_EXTERNAL_TESTS_FILTERS="test/experimental-tests-manifest.json" export NEXT_TEST_MODE=dev @@ -1002,6 +1004,7 @@ jobs: afterBuild: | export __NEXT_EXPERIMENTAL_PPR=true # for compatibility with the existing tests export __NEXT_EXPERIMENTAL_CACHE_COMPONENTS=true + export __NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION=true export NEXT_EXTERNAL_TESTS_FILTERS="test/experimental-tests-manifest.json" export NEXT_TEST_MODE=start diff --git a/packages/next/errors.json b/packages/next/errors.json index 542ec27051b0..228f1b85c5ec 100644 --- a/packages/next/errors.json +++ b/packages/next/errors.json @@ -808,5 +808,6 @@ "807": "Expected a %s response header.", "808": "Invalid binary HMR message: insufficient data (expected %s bytes, got %s)", "809": "Invalid binary HMR message of type %s", - "810": "React debug channel stream error" + "810": "React debug channel stream error", + "811": "Failed to load serialized config \"%s\"." } diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index c72704951a51..1c987144ed14 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -61,6 +61,7 @@ import { PRERENDER_MANIFEST, REACT_LOADABLE_MANIFEST, ROUTES_MANIFEST, + SERIALIZED_CONFIG_FILE, SERVER_DIRECTORY, SERVER_FILES_MANIFEST, STATIC_STATUS_PAGES, @@ -2723,6 +2724,36 @@ export default async function build( requiredServerFilesManifest ) + // The required-server-files manifest contains serialized config, which can + // be loaded by the prod server. However, when a custom distDir is set, + // the prod server will not know the distDir until loading the config. + // Therefore we write the serialized config to the same directory as the + // original config file. + if ( + config.output !== 'standalone' && + config.distDir !== '.next' && + // Use nullish coalescing (??) since we don't want to return when it's false. + (config.experimental?.serializeNextConfigForProduction ?? + // This flag is used to be enabled on the tests. + process.env + .__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true') + ) { + const serializedConfigPath = path.join( + // Write to the same directory as the original config file. + config.configFile ? path.dirname(config.configFile) : dir, + SERIALIZED_CONFIG_FILE + ) + await fs.writeFile( + serializedConfigPath, + JSON.stringify({ + // To match the format of required server files manifest. + version: 1, + config: requiredServerFilesManifest.config, + }) + ) + } + // we don't need to inline for turbopack build as // it will handle it's own caching separate of compile if (isGenerateMode && !isTurbopack) { diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index e7769094cd1c..a897c5517782 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -526,6 +526,7 @@ export const configSchema: zod.ZodType = z.lazy(() => ]) .optional(), optimizeRouterScrolling: z.boolean().optional(), + serializeNextConfigForProduction: z.boolean().optional(), }) .optional(), exportPathMap: z diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index d95a27c8a2ac..0f85b81ce588 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -818,6 +818,16 @@ export interface ExperimentalConfig { * instead of `{distDir}`. */ isolatedDevBuild?: boolean + + /** + * When enabled, the production server will use the serialized config file + * instead of the original config file. This can save the time of loading the + * config file, especially when you are using `next.config.ts`. When the `distDir` + * is set, the serialized config `next-config-serialized.json` will be written to + * the same directory as the original config file. This is because Next.js doesn't + * know the `distDir` until loading the config. + */ + serializeNextConfigForProduction?: boolean } export type ExportPathMap = { @@ -1495,6 +1505,10 @@ export const defaultConfig = Object.freeze({ browserDebugInfoInTerminal: false, optimizeRouterScrolling: false, isolatedDevBuild: false, + serializeNextConfigForProduction: + // This flag is used to be enabled on the tests. + process.env.__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true', }, htmlLimitedBots: undefined, bundlePagesRouterDependencies: false, diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 8a49cd739282..d7575f6cc304 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -1,5 +1,13 @@ -import { existsSync } from 'fs' -import { basename, extname, join, relative, isAbsolute, resolve } from 'path' +import { existsSync, readFileSync } from 'fs' +import { + basename, + extname, + join, + relative, + isAbsolute, + resolve, + dirname, +} from 'path' import { pathToFileURL } from 'url' import findUp from 'next/dist/compiled/find-up' import * as Log from '../build/output/log' @@ -9,6 +17,9 @@ import { PHASE_DEVELOPMENT_SERVER, PHASE_EXPORT, PHASE_PRODUCTION_BUILD, + PHASE_PRODUCTION_SERVER, + SERVER_FILES_MANIFEST, + SERIALIZED_CONFIG_FILE, type PHASE_TYPE, } from '../shared/lib/constants' import { defaultConfig, normalizeConfig } from './config-shared' @@ -1344,6 +1355,93 @@ export default async function loadConfig( return standaloneConfig } + let path: string | undefined + // During prod server, we can load from the serialized config. + if (phase === PHASE_PRODUCTION_SERVER) { + // Try load from ".next" dir since we don't know the + // distDir until loading the config. + try { + const possiblyServerFilesManifestPath = join( + dir, + '.next', + SERVER_FILES_MANIFEST + ) + if (existsSync(possiblyServerFilesManifestPath)) { + const parsed = JSON.parse( + readFileSync(possiblyServerFilesManifestPath, 'utf8') + ) + const config: NextConfigComplete = parsed.config + + if ( + // Don't return here and will eventually fall back to loading the config. + // Use nullish coalescing (??) since we don't want to return when it's false. + config?.experimental?.serializeNextConfigForProduction ?? + // This flag is used to be enabled on the tests. + process.env + .__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === 'true' + ) { + // Cache the config + configCache.set(cacheKey, { + config, + rawConfig: config, + configuredExperimentalFeatures: [], + }) + + return config + } + } + } catch { + // Continue to next option + } + + // If the custom distDir is set, we write the serialized config + // to the same directory as the original config file. + path = await findUp(CONFIG_FILES, { cwd: dir }) + const targetDir = path ? dirname(path) : dir + + // Even though serializeNextConfigForProduction might be disabled, we still need to check + // the existSync because there's no way to know if serializeNextConfigForProduction + // is disabled until we load the config. + const serializedConfigPath = join(targetDir, SERIALIZED_CONFIG_FILE) + try { + if (existsSync(serializedConfigPath)) { + const parsed = JSON.parse(readFileSync(serializedConfigPath, 'utf8')) + const config: NextConfigComplete = parsed.config + + if ( + // Don't return here and will eventually fall back to loading the config. + // Use nullish coalescing (??) since we don't want to return when it's false. + config?.experimental?.serializeNextConfigForProduction ?? + // This flag is used to be enabled on the tests. + process.env + .__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === 'true' + ) { + // Cache the config + configCache.set(cacheKey, { + config, + rawConfig: config, + configuredExperimentalFeatures: [], + }) + + return config + } + } + } catch (cause) { + if ( + // This flag is used to be enabled on the tests. + process.env.__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true' + ) { + throw new Error( + `Failed to load serialized config "${serializedConfigPath}".`, + { cause } + ) + } + } + + // Fall back to loading the config. + } + const curLog = silent ? { warn: () => {}, @@ -1389,7 +1487,7 @@ export default async function loadConfig( return config } - const path = await findUp(CONFIG_FILES, { cwd: dir }) + path ??= await findUp(CONFIG_FILES, { cwd: dir }) // If config file was found if (path?.length) { diff --git a/packages/next/src/shared/lib/constants.ts b/packages/next/src/shared/lib/constants.ts index f1ccf3e13476..c2f6314b7ba3 100644 --- a/packages/next/src/shared/lib/constants.ts +++ b/packages/next/src/shared/lib/constants.ts @@ -96,6 +96,7 @@ export const PRERENDER_MANIFEST = 'prerender-manifest.json' export const ROUTES_MANIFEST = 'routes-manifest.json' export const IMAGES_MANIFEST = 'images-manifest.json' export const SERVER_FILES_MANIFEST = 'required-server-files.json' +export const SERIALIZED_CONFIG_FILE = 'next-config-serialized.json' export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json' export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json' export const TURBOPACK_CLIENT_MIDDLEWARE_MANIFEST = diff --git a/test/e2e/app-dir/nx-handling/apps/next-nx-test/next.config.js b/test/e2e/app-dir/nx-handling/apps/next-nx-test/next.config.js index 70cc8d82b4b5..2223f3829b48 100644 --- a/test/e2e/app-dir/nx-handling/apps/next-nx-test/next.config.js +++ b/test/e2e/app-dir/nx-handling/apps/next-nx-test/next.config.js @@ -10,6 +10,13 @@ const nextConfig = { // Use this to set Nx-specific options // See: https://nx.dev/recipes/next/next-config-setup nx: {}, + experimental: { + // Disable because nx tries to copy the config to the dist dir + // and expect to load the config inside the dist dir again. + // In this case, the `distDir` will be relative to the original config, + // not the config inside the dist dir, and cause distDir path mismatch. + serializeNextConfigForProduction: false, + }, } const plugins = [ diff --git a/test/e2e/config-schema-check/index.test.ts b/test/e2e/config-schema-check/index.test.ts index 9e483ae0dcb3..cf9904d5c797 100644 --- a/test/e2e/config-schema-check/index.test.ts +++ b/test/e2e/config-schema-check/index.test.ts @@ -58,8 +58,16 @@ describe('next.config.js schema validating - invalid config', () => { expect(output).toContain('Invalid next.config.js options detected') expect(output).toContain('badKey') - // for next start and next build we both display the warnings - expect(warningTimes).toBe(isNextStart ? 2 : 1) + if ( + process.env.__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true' + ) { + // With serialized config, warnings only appear during build, not during start + expect(warningTimes).toBe(1) + } else { + // for next start and next build we both display the warnings + expect(warningTimes).toBe(isNextStart ? 2 : 1) + } return 'success' }, 'success') diff --git a/test/e2e/next-phase/index.test.ts b/test/e2e/next-phase/index.test.ts index 3ff81f18156f..95c0612b9123 100644 --- a/test/e2e/next-phase/index.test.ts +++ b/test/e2e/next-phase/index.test.ts @@ -23,10 +23,16 @@ describe('next-phase', () => { if (skipped) return it('should render page with next phase correctly', async () => { + const isSerializedNextConfigForProduction = + process.env.__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true' const phases = { dev: 'phase-development-server', build: 'phase-production-build', - start: 'phase-production-server', + start: isSerializedNextConfigForProduction + ? // Serialized next config will use the config from build. + 'phase-production-build' + : 'phase-production-server', } const currentPhase = isNextDev ? phases.dev : phases.build const nonExistedPhase = isNextDev ? phases.build : phases.dev @@ -40,7 +46,9 @@ describe('next-phase', () => { if (isNextDev) { expect(next.cliOutput).not.toContain(phases.start) } else { - expect(next.cliOutput).toContain(phases.start) + if (isSerializedNextConfigForProduction) { + expect(next.cliOutput).toContain(phases.start) + } } }) }) diff --git a/test/integration/config-experimental-warning/test/index.test.js b/test/integration/config-experimental-warning/test/index.test.js index fb7e3f2ee7e6..40d8e4fe8ba3 100644 --- a/test/integration/config-experimental-warning/test/index.test.js +++ b/test/integration/config-experimental-warning/test/index.test.js @@ -21,20 +21,25 @@ const experimentalHeader = ' - Experiments (use with caution):' let app async function collectStdoutFromDev(appDir) { let stdout = '' + let stderr = '' const port = await findPort() app = await launchApp(appDir, port, { onStdout(msg) { stdout += msg }, + onStderr(msg) { + stderr += msg + }, }) - return stdout + return { stdout, stderr } } async function collectStdoutFromBuild(appDir) { - const { stdout } = await nextBuild(appDir, [], { + const { stdout, stderr } = await nextBuild(appDir, [], { stdout: true, + stderr: true, }) - return stdout + return { stdout, stderr } } describe('Config Experimental Warning', () => { @@ -58,7 +63,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).not.toMatch(experimentalHeader) }) @@ -69,7 +74,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).not.toMatch(experimentalHeader) }) @@ -82,7 +87,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' ✓ workerThreads') }) @@ -96,7 +101,7 @@ describe('Config Experimental Warning', () => { }) `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' ✓ workerThreads') }) @@ -110,7 +115,7 @@ describe('Config Experimental Warning', () => { }) `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).not.toContain(experimentalHeader) expect(stdout).not.toContain('workerThreads') }) @@ -124,7 +129,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' ⨯ prerenderEarlyExit') }) @@ -138,7 +143,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' · cpus: 2') }) @@ -152,7 +157,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' · ppr: "incremental"') }) @@ -167,7 +172,7 @@ describe('Config Experimental Warning', () => { } `) - const stdout = await collectStdoutFromDev(appDir) + const { stdout } = await collectStdoutFromDev(appDir) expect(stdout).toContain(experimentalHeader) expect(stdout).toContain(' ✓ workerThreads') expect(stdout).toContain(' ✓ scrollRestoration') @@ -210,7 +215,7 @@ describe('Config Experimental Warning', () => { } } `) - const stdout = await collectStdoutFromBuild(appDir) + const { stdout } = await collectStdoutFromBuild(appDir) expect(stdout).toMatch(experimentalHeader) expect(stdout).toMatch(' · cpus: 2') expect(stdout).toMatch(' ✓ workerThreads') @@ -219,7 +224,8 @@ describe('Config Experimental Warning', () => { expect(stdout).toMatch(' ✓ parallelServerCompiles') }) - it('should show unrecognized experimental features in warning but not in start log experiments section', async () => { + // In prod, it will load a serialized config, so the warning will not appear during start. + it('should show unrecognized experimental features in warning but not in start log experiments section during build', async () => { configFile.write(` module.exports = { experimental: { @@ -228,7 +234,8 @@ describe('Config Experimental Warning', () => { } `) - await collectStdoutFromBuild(appDir) + const { stderr: buildStderr } = await collectStdoutFromBuild(appDir) + const port = await findPort() let stdout = '' let stderr = '' @@ -242,13 +249,27 @@ describe('Config Experimental Warning', () => { }) await check(() => { - const cliOutput = stripAnsi(stdout) - const cliOutputErr = stripAnsi(stderr) - expect(cliOutput).not.toContain(experimentalHeader) - expect(cliOutputErr).toContain( + // TODO: experimentalHeader shows on build, likely not validating properly. + // expect(stripAnsi(buildStdout)).not.toContain(experimentalHeader) + expect(stripAnsi(buildStderr)).toContain( `Unrecognized key(s) in object: 'appDir' at "experimental"` ) }) + + // For serialized config, the warning will not appear during start. + if ( + process.env + .__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION !== 'true' + ) { + await check(() => { + const cliOutput = stripAnsi(stdout) + const cliOutputErr = stripAnsi(stderr) + expect(cliOutput).not.toContain(experimentalHeader) + expect(cliOutputErr).toContain( + `Unrecognized key(s) in object: 'appDir' at "experimental"` + ) + }) + } }) } ) diff --git a/test/integration/custom-server/test/index.test.js b/test/integration/custom-server/test/index.test.js index 3e67277192a8..eb54720cd25c 100644 --- a/test/integration/custom-server/test/index.test.js +++ b/test/integration/custom-server/test/index.test.js @@ -145,8 +145,21 @@ describe.each([ 'production mode', () => { beforeAll(async () => { - await nextBuild(appDir) - await startServer({ GENERATE_ETAGS: 'true', NODE_ENV: 'production' }) + if ( + process.env + .__NEXT_EXPERIMENTAL_SERIALIZE_NEXT_CONFIG_FOR_PRODUCTION === + 'true' + ) { + // Set env during build for serialized next config. + await nextBuild(appDir, [], { env: { GENERATE_ETAGS: 'true' } }) + await startServer({ NODE_ENV: 'production' }) + } else { + await nextBuild(appDir) + await startServer({ + GENERATE_ETAGS: 'true', + NODE_ENV: 'production', + }) + } }) afterAll(() => killApp(server)) diff --git a/test/production/app-dir/next-config-serialized/app/layout.tsx b/test/production/app-dir/next-config-serialized/app/layout.tsx new file mode 100644 index 000000000000..888614deda3b --- /dev/null +++ b/test/production/app-dir/next-config-serialized/app/layout.tsx @@ -0,0 +1,8 @@ +import { ReactNode } from 'react' +export default function Root({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/production/app-dir/next-config-serialized/app/page.tsx b/test/production/app-dir/next-config-serialized/app/page.tsx new file mode 100644 index 000000000000..71e8787438c0 --- /dev/null +++ b/test/production/app-dir/next-config-serialized/app/page.tsx @@ -0,0 +1,3 @@ +export default function Page() { + return

hello world {process.env.foo}

+} diff --git a/test/production/app-dir/next-config-serialized/next-config-serialized.test.ts b/test/production/app-dir/next-config-serialized/next-config-serialized.test.ts new file mode 100644 index 000000000000..cecaada1dbda --- /dev/null +++ b/test/production/app-dir/next-config-serialized/next-config-serialized.test.ts @@ -0,0 +1,60 @@ +import { nextTestSetup } from 'e2e-utils' +import { retry } from 'next-test-utils' + +describe('next-config-serialized', () => { + const { next, skipped } = nextTestSetup({ + files: __dirname, + skipStart: true, + skipDeployment: true, + }) + + if (skipped) { + return + } + + it('should use .next/required-server-files.json when distDir is .next', async () => { + await next.build() + expect(await next.hasFile('.next/required-server-files.json')).toBe(true) + + // Rename the config file so it can't be loaded. + await next.renameFile('next.config.js', 'next.config.noop.js') + await retry(async () => { + expect(await next.hasFile('next.config.noop.js')).toBe(true) + }) + + await next.start({ skipBuild: true }) + + const browser = await next.browser('/') + expect(await browser.elementByCss('p').text()).toBe('hello world foo') + + // Restore the file for the next case. + await next.renameFile('next.config.noop.js', 'next.config.js') + await next.stop() + }) + + it('should use next-config-serialized.json when distDir is not .next', async () => { + await next.patchFile('next.config.js', (content) => { + return content.replace(`// distDir: 'out',`, `distDir: 'out',`) + }) + + await next.build() + expect(await next.hasFile('out/required-server-files.json')).toBe(true) + // next-config-serialized.json created next to next.config.js + expect(await next.hasFile('next-config-serialized.json')).toBe(true) + + // Rename the config file so it can't be loaded. + await next.renameFile('next.config.js', 'next.config.noop.js') + await retry(async () => { + expect(await next.hasFile('next.config.noop.js')).toBe(true) + }) + + await next.start({ skipBuild: true }) + + const browser = await next.browser('/') + expect(await browser.elementByCss('p').text()).toBe('hello world foo') + + // Restore the file for the next case. + await next.renameFile('next.config.noop.js', 'next.config.js') + await next.stop() + }) +}) diff --git a/test/production/app-dir/next-config-serialized/next.config.js b/test/production/app-dir/next-config-serialized/next.config.js new file mode 100644 index 000000000000..81a6ab13bae1 --- /dev/null +++ b/test/production/app-dir/next-config-serialized/next.config.js @@ -0,0 +1,14 @@ +/** + * @type {import('next').NextConfig} + */ +const nextConfig = { + // distDir: 'out', + env: { + foo: 'foo', + }, + experimental: { + serializeNextConfigForProduction: true, + }, +} + +module.exports = nextConfig