server: bundle vendored react (#55362)

## What

This PR changes Next.js to bundle its vendored React libraries so that the App Router pages can use those built-in versions.

## Why

Next.js supports both Pages and App Router and we've gone through a lot of iteration to make sure that Next.js stays flexible wrt to the version of React used: in Pages, we want to use the React provided by the user and in the App Router, to be able to use it, we need to use the canary version of React, which we've built into Next.js for convenience.

The problem stems from the fact that you can't run two different instances of React (by design).

Previously we have a dual worker setup, where we would separate completely each Next.js versions (App and Pages) so that they would not overlap with each other, however this approach was not great performance and memory wise.

We've recently tried using an ESM loader and a single process, but this change would still opt you into the React canary version if you had an app page, which breaks some assumptions.

## How

A list of the changes in this PR:

### New versions of the Next.js runtime

Since we now compile a runtime per type of page (app/route/api/pages), in order to bundle the two versions of React that we vendored, we introduced a new type of bundle suffixed by `-experimental`. This bundle will have the bleeding edge React needed for Server Actions and Next.js will opt you in into that runtime automatically.

For internal contributors, it means that we now run a compiler for 10 subparts of Next.js:
- next_bundle_server
- next_bundle_pages_prod
- next_bundle_pages_turbo
- next_bundle_pages_dev
- next_bundle_app_turbo_experimental
- next_bundle_app_prod
- next_bundle_app_prod_experimental
- next_bundle_app_turbo
- next_bundle_app_dev_experimental
- next_bundle_app_dev

![image](https://github.com/vercel/next.js/assets/11064311/f340417d-845e-45b9-8e86-5b287a295c82)

### Simplified require-hook

Since the versions of React are correctly re-routed at build time for app pages, we don't need the require hook anymore

### Turbopack changes

The bundling logic in Turbopack has been addressed to properly follow the new logic

### Changes to the shared contexts system

Some context files need to have a shared instance between the rendering runtime and the user code, like the one that powers the `next/image` component. In general, the aliasing setup takes care of that but we need the require hook for code that is not compiled to reroute to the correct runtime. This only happens for pages node_modules.

A new Turbopack resolving plugin has been added to handle that logic in Turbopack.

### Misc changes

- `runtime-config` (that powers `next/config`) has been converted to an `.external` file, as it should have been
- there are some rules that have been added to the aliases to support the usage of `react-dom/server` in a server-components. We can do that now since the runtime takes care of separating the versions of React.



Co-authored-by: JJ Kasper <[email protected]>

Author: feedthejim

packages/next-swc/crates/next-core/src/next_import_map.rs

@@ -246,10 +246,7 @@ pub async fn get_next_server_import_map(
         | ServerContextType::AppRSC { .. }
         | ServerContextType::AppRoute { .. } => {
             match mode {
-                NextMode::Build => {
-                    import_map.insert_wildcard_alias("next/dist/server/", external);
-                    import_map.insert_wildcard_alias("next/dist/shared/", external);
-                }
+                NextMode::Build => {}
                 NextMode::DevServer => {
                     // The sandbox can't be bundled and needs to be external
                     import_map.insert_exact_alias("next/dist/server/web/sandbox", external);
@@ -431,61 +428,138 @@ async fn insert_next_server_special_aliases(
             );
         }
         (_, ServerContextType::PagesData { .. }) => {}
-        // In development, we *always* use the bundled version of React, even in
-        // SSR, since we're bundling Next.js alongside it.
-        (NextMode::DevServer, ServerContextType::AppSSR { app_dir }) => {
+        // the logic closely follows the one in createRSCAliases in webpack-config.ts
+        (
+            NextMode::DevServer | NextMode::Build | NextMode::Development,
+            ServerContextType::AppSSR { app_dir },
+        ) => {
             import_map.insert_exact_alias(
                 "@opentelemetry/api",
                 // TODO(WEB-625) this actually need to prefer the local version of
                 // @opentelemetry/api
                 request_to_import_mapping(app_dir, "next/dist/compiled/@opentelemetry/api"),
             );
             import_map.insert_exact_alias(
-                "react",
-                passthrough_external_if_node(app_dir, "next/dist/compiled/react"),
+                "styled-jsx",
+                passthrough_external_if_node(app_dir, "next/dist/compiled/styled-jsx"),
             );
             import_map.insert_wildcard_alias(
-                "react/",
-                passthrough_external_if_node(app_dir, "next/dist/compiled/react/*"),
+                "styled-jsx/",
+                passthrough_external_if_node(app_dir, "next/dist/compiled/styled-jsx/*"),
+            );
+            import_map.insert_exact_alias(
+                "react/jsx-runtime",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react/jsx-runtime",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/shared/\
+                             react-jsx-runtime"
+                        }
+                    },
+                ),
+            );
+            import_map.insert_exact_alias(
+                "react/jsx-dev-runtime",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react/jsx-dev-runtime",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/shared/\
+                             react-jsx-dev-runtime"
+                        }
+                    },
+                ),
+            );
+            import_map.insert_exact_alias(
+                "react",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/react"
+                        }
+                    },
+                ),
             );
             import_map.insert_exact_alias(
                 "react-dom",
-                passthrough_external_if_node(
+                request_to_import_mapping(
                     app_dir,
-                    "next/dist/compiled/react-dom/server-rendering-stub.js",
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react-dom",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/react-dom"
+                        }
+                    },
                 ),
             );
-            import_map.insert_wildcard_alias(
-                "react-dom/",
-                passthrough_external_if_node(app_dir, "next/dist/compiled/react-dom/*"),
+            import_map.insert_exact_alias(
+                "react-server-dom-webpack/client.edge",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => {
+                            "next/dist/compiled/react-server-dom-webpack/client.edge"
+                        }
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/\
+                             react-server-dom-webpack-client-edge"
+                        }
+                    },
+                ),
             );
+            // some code also imports react-server-dom-webpack/client on the server
+            // it should never run so it's fine to just point it to the same place as
+            // react-server-dom-webpack/client.edge
             import_map.insert_exact_alias(
-                "styled-jsx",
-                passthrough_external_if_node(app_dir, "next/dist/compiled/styled-jsx"),
+                "react-server-dom-webpack/client",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => {
+                            "next/dist/compiled/react-server-dom-webpack/client.edge"
+                        }
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/\
+                             react-server-dom-webpack-client-edge"
+                        }
+                    },
+                ),
             );
-            import_map.insert_wildcard_alias(
-                "styled-jsx/",
-                passthrough_external_if_node(app_dir, "next/dist/compiled/styled-jsx/*"),
+            // not essential but we're providing this alias for people who might use it.
+            // A note here is that this will point toward the ReactDOMServer on the SSR
+            // layer TODO: add the rests
+            import_map.insert_exact_alias(
+                "react-dom/server",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react-dom/server.edge",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/\
+                             react-dom-server-edge"
+                        }
+                    },
+                ),
             );
-            import_map.insert_wildcard_alias(
-                "react-server-dom-webpack/",
-                passthrough_external_if_node(
+            import_map.insert_exact_alias(
+                "react-dom/server.edge",
+                request_to_import_mapping(
                     app_dir,
-                    "next/dist/compiled/react-server-dom-webpack/*",
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react-dom/server.edge",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/\
+                             react-dom-server-edge"
+                        }
+                    },
                 ),
             );
         }
-
-        // NOTE(alexkirsz) This logic maps loosely to
-        // `next.js/packages/next/src/build/webpack-config.ts`, where:
-        //
-        // ## RSC
-        //
-        // * always bundles
-        // * maps react -> react/shared-subset (through the "react-server" exports condition)
-        // * maps react-dom -> react-dom/server-rendering-stub
-        // * passes through (react|react-dom|react-server-dom-webpack)/(.*) to
-        //   next/dist/compiled/$1/$2
         (
             NextMode::Build | NextMode::Development | NextMode::DevServer,
             ServerContextType::AppRSC { app_dir, .. } | ServerContextType::AppRoute { app_dir },
@@ -496,72 +570,103 @@ async fn insert_next_server_special_aliases(
                 // @opentelemetry/api
                 request_to_import_mapping(app_dir, "next/dist/compiled/@opentelemetry/api"),
             );
-            if matches!(ty, ServerContextType::AppRSC { .. }) {
-                import_map.insert_exact_alias(
-                    "react",
-                    request_to_import_mapping(
-                        app_dir,
-                        "next/dist/compiled/react/react.shared-subset",
-                    ),
-                );
-            } else {
-                import_map.insert_exact_alias(
-                    "react",
-                    request_to_import_mapping(app_dir, "next/dist/compiled/react"),
-                );
-            }
+
             import_map.insert_exact_alias(
-                "react-dom",
+                "react/jsx-runtime",
                 request_to_import_mapping(
                     app_dir,
-                    "next/dist/compiled/react-dom/server-rendering-stub",
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react/jsx-runtime",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/shared/\
+                             react-jsx-runtime"
+                        }
+                    },
                 ),
             );
-            for (wildcard_alias, request) in [
-                ("react/", "next/dist/compiled/react/*"),
-                ("react-dom/", "next/dist/compiled/react-dom/*"),
-                (
-                    "react-server-dom-webpack/",
-                    "next/dist/compiled/react-server-dom-webpack/*",
+            import_map.insert_exact_alias(
+                "react/jsx-dev-runtime",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react/jsx-dev-runtime",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/shared/\
+                             react-jsx-dev-runtime"
+                        }
+                    },
                 ),
-            ] {
-                import_map.insert_wildcard_alias(
-                    wildcard_alias,
-                    request_to_import_mapping(app_dir, request),
-                );
-            }
-        }
-        // ## SSR
-        //
-        // * always uses externals, to ensure we're using the same React instance as the Next.js
-        //   runtime
-        // * maps react-dom -> react-dom/server-rendering-stub
-        // * passes through react and (react|react-dom|react-server-dom-webpack)/(.*) to
-        //   next/dist/compiled/react and next/dist/compiled/$1/$2 resp.
-        (NextMode::Build | NextMode::Development, ServerContextType::AppSSR { app_dir }) => {
+            );
             import_map.insert_exact_alias(
                 "react",
-                external_if_node(app_dir, "next/dist/compiled/react"),
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/rsc/react"
+                        }
+                    },
+                ),
             );
             import_map.insert_exact_alias(
                 "react-dom",
-                external_if_node(
+                request_to_import_mapping(
                     app_dir,
-                    "next/dist/compiled/react-dom/server-rendering-stub",
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react-dom",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/rsc/react-dom"
+                        }
+                    },
                 ),
             );
-
-            for (wildcard_alias, request) in [
-                ("react/", "next/dist/compiled/react/*"),
-                ("react-dom/", "next/dist/compiled/react-dom/*"),
-                (
-                    "react-server-dom-webpack/",
-                    "next/dist/compiled/react-server-dom-webpack/*",
+            import_map.insert_exact_alias(
+                "react-server-dom-webpack/server.edge",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => {
+                            "next/dist/compiled/react-server-dom-webpack/server.edge"
+                        }
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/rsc/\
+                             react-server-dom-webpack-server-edge"
+                        }
+                    },
                 ),
-            ] {
-                let import_mapping = external_if_node(app_dir, request);
-                import_map.insert_wildcard_alias(wildcard_alias, import_mapping);
-            }
+            );
+            import_map.insert_exact_alias(
+                "react-server-dom-webpack/server.node",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => {
+                            "next/dist/compiled/react-server-dom-webpack/server.node"
+                        }
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/rsc/\
+                             react-server-dom-webpack-server-node"
+                        }
+                    },
+                ),
+            );
+            // not essential but we're providing this alias for people who might use it.
+            // A note here is that this will point toward the ReactDOMServer on the SSR
+            // layer TODO: add the rests
+            import_map.insert_exact_alias(
+                "react-dom/server.edge",
+                request_to_import_mapping(
+                    app_dir,
+                    match runtime {
+                        NextRuntime::Edge => "next/dist/compiled/react-dom/server.edge",
+                        NextRuntime::NodeJs => {
+                            "next/dist/server/future/route-modules/app-page/vendored/ssr/\
+                             react-dom-server-edge"
+                        }
+                    },
+                ),
+            );
         }
         (_, ServerContextType::Middleware) => {}
     }

packages/next-swc/crates/next-core/src/next_server/context.rs

@@ -48,7 +48,7 @@ use crate::{
     next_shared::{
         resolve::{
             ModuleFeatureReportResolvePlugin, NextExternalResolvePlugin,
-            UnsupportedModulesResolvePlugin,
+            NextNodeSharedRuntimeResolvePlugin, UnsupportedModulesResolvePlugin,
         },
         transforms::{
             emotion::get_emotion_transform_plugin, get_relay_transform_plugin,
@@ -125,6 +125,8 @@ pub async fn get_server_resolve_options_context(
     );
 
     let next_external_plugin = NextExternalResolvePlugin::new(project_path);
+    let next_node_shared_runtime_plugin =
+        NextNodeSharedRuntimeResolvePlugin::new(project_path, Value::new(ty));
 
     let plugins = match ty {
         ServerContextType::Pages { .. } | ServerContextType::PagesData { .. } => {
@@ -133,6 +135,7 @@ pub async fn get_server_resolve_options_context(
                 Vc::upcast(external_cjs_modules_plugin),
                 Vc::upcast(unsupported_modules_resolve_plugin),
                 Vc::upcast(next_external_plugin),
+                Vc::upcast(next_node_shared_runtime_plugin),
             ]
         }
         ServerContextType::AppSSR { .. }
@@ -144,6 +147,7 @@ pub async fn get_server_resolve_options_context(
                 Vc::upcast(server_component_externals_plugin),
                 Vc::upcast(unsupported_modules_resolve_plugin),
                 Vc::upcast(next_external_plugin),
+                Vc::upcast(next_node_shared_runtime_plugin),
             ]
         }
     };
@@ -175,7 +179,8 @@ fn defines(mode: NextMode) -> CompileTimeDefines {
         process.turbopack = true,
         process.env.NODE_ENV = mode.node_env(),
         process.env.__NEXT_CLIENT_ROUTER_FILTER_ENABLED = false,
-        process.env.NEXT_RUNTIME = "nodejs"
+        process.env.NEXT_RUNTIME = "nodejs",
+        process.env.__NEXT_EXPERIMENTAL_REACT = false,
     )
     // TODO(WEB-937) there are more defines needed, see
     // packages/next/src/build/webpack-config.ts