Initial implementation of statically optimized flight data of server component pages (#35619)

Part of #31506 and #34179. This PR ensures that in the `nodejs` runtime, the flight data is statically stored as a JSON file if possible. Most of the touched code is related to conditions of static/SSG/SSR when runtime and/or RSC is involved.

## Bug

- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `yarn lint`

Author: shuding

packages/next/build/index.ts

@@ -99,6 +99,7 @@ import {
   copyTracedFiles,
   isReservedPage,
   isCustomErrorPage,
+  isFlightPage,
 } from './utils'
 import getBaseWebpackConfig from './webpack-config'
 import { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
@@ -162,7 +163,6 @@ export default async function build(
     // using React 18 or experimental.
     const hasReactRoot = shouldUseReactRoot()
     const hasConcurrentFeatures = hasReactRoot
-
     const hasServerComponents =
       hasReactRoot && !!config.experimental.serverComponents
 
@@ -288,6 +288,7 @@ export default async function build(
       .traceAsyncFn(() => collectPages(pagesDir, config.pageExtensions))
     // needed for static exporting since we want to replace with HTML
     // files
+
     const allStaticPages = new Set<string>()
     let allPageInfos = new Map<string, PageInfo>()
 
@@ -963,6 +964,7 @@ export default async function build(
 
             let isSsg = false
             let isStatic = false
+            let isServerComponent = false
             let isHybridAmp = false
             let ssgPageRoutes: string[] | null = null
             let isMiddlewareRoute = !!page.match(MIDDLEWARE_ROUTE)
@@ -976,6 +978,12 @@ export default async function build(
               ? await getPageRuntime(join(pagesDir, pagePath), config)
               : undefined
 
+            if (hasServerComponents && pagePath) {
+              if (isFlightPage(config, pagePath)) {
+                isServerComponent = true
+              }
+            }
+
             if (
               !isMiddlewareRoute &&
               !isReservedPage(page) &&
@@ -1045,11 +1053,16 @@ export default async function build(
                   serverPropsPages.add(page)
                 } else if (
                   workerResult.isStatic &&
-                  !workerResult.hasFlightData &&
+                  !isServerComponent &&
                   (await customAppGetInitialPropsPromise) === false
                 ) {
                   staticPages.add(page)
                   isStatic = true
+                } else if (isServerComponent) {
+                  // This is a static server component page that doesn't have
+                  // gSP or gSSP. We still treat it as a SSG page.
+                  ssgPages.add(page)
+                  isSsg = true
                 }
 
                 if (hasPages404 && page === '/404') {

packages/next/build/utils.ts

@@ -859,7 +859,6 @@ export async function isPageStatic(
   isStatic?: boolean
   isAmpOnly?: boolean
   isHybridAmp?: boolean
-  hasFlightData?: boolean
   hasServerProps?: boolean
   hasStaticProps?: boolean
   prerenderRoutes?: string[]
@@ -882,7 +881,6 @@ export async function isPageStatic(
         throw new Error('INVALID_DEFAULT_EXPORT')
       }
 
-      const hasFlightData = !!(mod as any).__next_rsc__
       const hasGetInitialProps = !!(Comp as any).getInitialProps
       const hasStaticProps = !!mod.getStaticProps
       const hasStaticPaths = !!mod.getStaticPaths
@@ -970,19 +968,14 @@ export async function isPageStatic(
       const isNextImageImported = (global as any).__NEXT_IMAGE_IMPORTED
       const config: PageConfig = mod.pageConfig
       return {
-        isStatic:
-          !hasStaticProps &&
-          !hasGetInitialProps &&
-          !hasServerProps &&
-          !hasFlightData,
+        isStatic: !hasStaticProps && !hasGetInitialProps && !hasServerProps,
         isHybridAmp: config.amp === 'hybrid',
         isAmpOnly: config.amp === true,
         prerenderRoutes,
         prerenderFallback,
         encodedPrerenderRoutes,
         hasStaticProps,
         hasServerProps,
-        hasFlightData,
         isNextImageImported,
         traceIncludes: config.unstable_includeFiles || [],
         traceExcludes: config.unstable_excludeFiles || [],

packages/next/build/webpack/loaders/next-flight-server-loader.ts

@@ -41,6 +41,8 @@ async function parseModuleInfo({
   source: string
   imports: string
   isEsm: boolean
+  __N_SSP: boolean
+  pageRuntime: 'edge' | 'nodejs' | null
 }> {
   const ast = await parse(source, {
     filename: resourcePath,
@@ -50,12 +52,15 @@ async function parseModuleInfo({
   let transformedSource = ''
   let lastIndex = 0
   let imports = ''
+  let __N_SSP = false
+  let pageRuntime = null
+
   const isEsm = type === 'Module'
 
   for (let i = 0; i < body.length; i++) {
     const node = body[i]
     switch (node.type) {
-      case 'ImportDeclaration': {
+      case 'ImportDeclaration':
         const importSource = node.source.value
         if (!isClientCompilation) {
           // Server compilation for .server.js.
@@ -112,7 +117,32 @@ async function parseModuleInfo({
 
         lastIndex = node.source.span.end
         break
-      }
+      case 'ExportDeclaration':
+        if (isClientCompilation) {
+          // Keep `__N_SSG` and `__N_SSP` exports.
+          if (node.declaration?.type === 'VariableDeclaration') {
+            for (const declaration of node.declaration.declarations) {
+              if (declaration.type === 'VariableDeclarator') {
+                if (declaration.id?.type === 'Identifier') {
+                  const value = declaration.id.value
+                  if (value === '__N_SSP') {
+                    __N_SSP = true
+                  } else if (value === 'config') {
+                    const props = declaration.init.properties
+                    const runtimeKeyValue = props.find(
+                      (prop: any) => prop.key.value === 'runtime'
+                    )
+                    const runtime = runtimeKeyValue?.value?.value
+                    if (runtime === 'nodejs' || runtime === 'edge') {
+                      pageRuntime = runtime
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        break
       default:
         break
     }
@@ -122,7 +152,7 @@ async function parseModuleInfo({
     transformedSource += source.substring(lastIndex)
   }
 
-  return { source: transformedSource, imports, isEsm }
+  return { source: transformedSource, imports, isEsm, __N_SSP, pageRuntime }
 }
 
 export default async function transformSource(
@@ -161,6 +191,8 @@ export default async function transformSource(
     source: transformedSource,
     imports,
     isEsm,
+    __N_SSP,
+    pageRuntime,
   } = await parseModuleInfo({
     resourcePath,
     source,
@@ -190,7 +222,20 @@ export default async function transformSource(
   }
 
   if (isClientCompilation) {
-    rscExports['default'] = 'function RSC() {}'
+    rscExports.default = 'function RSC() {}'
+
+    if (pageRuntime === 'edge') {
+      // Currently for the Edge runtime, we treat all RSC pages as SSR pages.
+      rscExports.__N_SSP = 'true'
+    } else {
+      if (__N_SSP) {
+        rscExports.__N_SSP = 'true'
+      } else {
+        // Server component pages are always considered as SSG by default because
+        // the flight data is needed for client navigation.
+        rscExports.__N_SSG = 'true'
+      }
+    }
   }
 
   const output = transformedSource + '\n' + buildExports(rscExports, isEsm)

packages/next/client/index.tsx

@@ -547,14 +547,17 @@ function renderReactElement(
 
   const reactEl = fn(shouldHydrate ? markHydrateComplete : markRenderComplete)
   if (process.env.__NEXT_REACT_ROOT) {
-    const ReactDOMClient = require('react-dom/client')
     if (!reactRoot) {
       // Unlike with createRoot, you don't need a separate root.render() call here
-      reactRoot = (ReactDOMClient as any).hydrateRoot(domEl, reactEl)
+      const ReactDOMClient = require('react-dom/client')
+      reactRoot = ReactDOMClient.hydrateRoot(domEl, reactEl)
       // TODO: Remove shouldHydrate variable when React 18 is stable as it can depend on `reactRoot` existing
       shouldHydrate = false
     } else {
-      reactRoot.render(reactEl)
+      const startTransition = (React as any).startTransition
+      startTransition(() => {
+        reactRoot.render(reactEl)
+      })
     }
   } else {
     // The check for `.hydrate` is there to support React alternatives like preact
@@ -675,6 +678,7 @@ if (process.env.__NEXT_RSC) {
 
   const {
     createFromFetch,
+    createFromReadableStream,
   } = require('next/dist/compiled/react-server-dom-webpack')
 
   const encoder = new TextEncoder()
@@ -769,20 +773,19 @@ if (process.env.__NEXT_RSC) {
           nextServerDataRegisterWriter(controller)
         },
       })
-      response = createFromFetch(Promise.resolve({ body: readable }))
+      response = createFromReadableStream(readable)
     } else {
-      const fetchPromise = serialized
-        ? (() => {
-            const readable = new ReadableStream({
-              start(controller) {
-                controller.enqueue(new TextEncoder().encode(serialized))
-                controller.close()
-              },
-            })
-            return Promise.resolve({ body: readable })
-          })()
-        : fetchFlight(getCacheKey())
-      response = createFromFetch(fetchPromise)
+      if (serialized) {
+        const readable = new ReadableStream({
+          start(controller) {
+            controller.enqueue(new TextEncoder().encode(serialized))
+            controller.close()
+          },
+        })
+        response = createFromReadableStream(readable)
+      } else {
+        response = createFromFetch(fetchFlight(getCacheKey()))
+      }
     }
 
     rscCache.set(cacheKey, response)
@@ -800,16 +803,16 @@ if (process.env.__NEXT_RSC) {
       rscCache.delete(cacheKey)
     })
     const response = useServerResponse(cacheKey, serialized)
-    const root = response.readRoot()
-    return root
+    return response.readRoot()
   }
 
   RSCComponent = (props: any) => {
     const cacheKey = getCacheKey()
-    const { __flight_serialized__ } = props
+    const { __flight__ } = props
     const [, dispatch] = useState({})
     const startTransition = (React as any).startTransition
     const rerender = () => dispatch({})
+
     // If there is no cache, or there is serialized data already
     function refreshCache(nextProps?: any) {
       startTransition(() => {
@@ -825,7 +828,7 @@ if (process.env.__NEXT_RSC) {
 
     return (
       <RefreshContext.Provider value={refreshCache}>
-        <ServerRoot cacheKey={cacheKey} serialized={__flight_serialized__} />
+        <ServerRoot cacheKey={cacheKey} serialized={__flight__} />
       </RefreshContext.Provider>
     )
   }

packages/next/client/page-loader.ts

@@ -133,21 +133,21 @@ export default class PageLoader {
     href,
     asPath,
     ssg,
-    rsc,
+    flight,
     locale,
   }: {
     href: string
     asPath: string
     ssg?: boolean
-    rsc?: boolean
+    flight?: boolean
     locale?: string | false
   }): string {
     const { pathname: hrefPathname, query, search } = parseRelativeUrl(href)
     const { pathname: asPathname } = parseRelativeUrl(asPath)
     const route = normalizeRoute(hrefPathname)
 
     const getHrefForSlug = (path: string) => {
-      if (rsc) {
+      if (flight) {
         return path + search + (search ? `&` : '?') + '__flight__=1'
       }
 

packages/next/server/base-server.ts

@@ -1124,14 +1124,25 @@ export default abstract class Server {
     const isLikeServerless =
       typeof components.ComponentMod === 'object' &&
       typeof (components.ComponentMod as any).renderReqToHTML === 'function'
-    const isSSG = !!components.getStaticProps
     const hasServerProps = !!components.getServerSideProps
     const hasStaticPaths = !!components.getStaticPaths
     const hasGetInitialProps = !!components.Component?.getInitialProps
+    const isServerComponent = !!components.ComponentMod?.__next_rsc__
+    const isSSG =
+      !!components.getStaticProps ||
+      // For static server component pages, we currently always consider them
+      // as SSG since we also need to handle the next data (flight JSON).
+      (isServerComponent &&
+        !hasServerProps &&
+        !hasGetInitialProps &&
+        !process.browser)
 
     // Toggle whether or not this is a Data request
-    const isDataReq = !!query._nextDataReq && (isSSG || hasServerProps)
+    const isDataReq =
+      !!query._nextDataReq && (isSSG || hasServerProps || isServerComponent)
+
     delete query._nextDataReq
+
     // Don't delete query.__flight__ yet, it still needs to be used in renderToHTML later
     const isFlightRequest = Boolean(
       this.serverComponentManifest && query.__flight__
@@ -1290,8 +1301,8 @@ export default abstract class Server {
     }
 
     let ssgCacheKey =
-      isPreviewMode || !isSSG || opts.supportsDynamicHTML
-        ? null // Preview mode and manual revalidate bypasses the cache
+      isPreviewMode || !isSSG || opts.supportsDynamicHTML || isFlightRequest
+        ? null // Preview mode, manual revalidate, flight request can bypass the cache
         : `${locale ? `/${locale}` : ''}${
             (pathname === '/' || resolvedUrlPathname === '/') && locale
               ? ''
@@ -1602,7 +1613,10 @@ export default abstract class Server {
       if (isDataReq) {
         return {
           type: 'json',
-          body: RenderResult.fromStatic(JSON.stringify(cachedData.props)),
+          body: RenderResult.fromStatic(
+            // @TODO: Handle flight data.
+            JSON.stringify(cachedData.props)
+          ),
           revalidateOptions,
         }
       } else {

packages/next/server/next-server.ts

@@ -680,6 +680,7 @@ export default class NextNodeServer extends BaseServer {
                   _nextDataReq: query._nextDataReq,
                   __nextLocale: query.__nextLocale,
                   __nextDefaultLocale: query.__nextDefaultLocale,
+                  __flight__: query.__flight__,
                 } as NextParsedUrlQuery)
               : query),
             ...(params || {}),