refactor(lazy-compilation): use POST request to transfer ids of active modules

packages/rspack-test-tools/src/runner/web/index.ts

@@ -175,7 +175,12 @@ export class WebRunner extends NodeRunner {
   protected createBaseModuleScope() {
     const moduleScope = super.createBaseModuleScope();
     moduleScope.EventSource = EventSource;
-    moduleScope.fetch = async (url: string) => {
+    moduleScope.fetch = async (url: string, options: any) => {
+      // For Lazy Compilation Proxy the POST request to the real dev server.
+      if (options?.method === 'POST') {
+        return fetch(url, options as any);
+      }
+
       try {
         const filePath = this.urlToPath(url);
         this.log(`fetch: ${url} -> ${filePath}`);

packages/rspack/etc/core.api.md

@@ -4289,7 +4289,7 @@ interface LabeledStatement extends Node_4, HasSpan {
 // @public
 export type Layer = string | null;
 
-// @public
+// @public (undocumented)
 export const lazyCompilationMiddleware: (compiler: Compiler | MultiCompiler) => MiddlewareHandler;
 
 // @public

packages/rspack/hot/lazy-compilation-web.js

@@ -1,44 +1,70 @@
-if (typeof EventSource !== 'function') {
+if (typeof XMLHttpRequest === 'undefined') {
   throw new Error(
-    "Environment doesn't support lazy compilation (requires EventSource)",
+    "Environment doesn't support lazy compilation (requires XMLHttpRequest)",
   );
 }
 
 var urlBase = decodeURIComponent(__resourceQuery.slice(1));
-/** @type {EventSource | undefined} */
-var activeEventSource;
 var compiling = new Set();
 var errorHandlers = new Set();
 
-var updateEventSource = function updateEventSource() {
-  if (activeEventSource) activeEventSource.close();
-  if (compiling.size) {
-    activeEventSource = new EventSource(
-      urlBase +
-        Array.from(compiling, function (module) {
-          return encodeURIComponent(module);
-        }).join('@'),
-    );
-    /**
-     * @this {EventSource}
-     * @param {Event & { message?: string, filename?: string, lineno?: number, colno?: number, error?: Error }} event event
-     */
-    activeEventSource.onerror = function (event) {
-      errorHandlers.forEach(function (onError) {
-        onError(
-          new Error(
-            'Problem communicating active modules to the server' +
-              (event.message ? `: ${event.message} ` : '') +
-              (event.filename ? `: ${event.filename} ` : '') +
-              (event.lineno ? `: ${event.lineno} ` : '') +
-              (event.colno ? `: ${event.colno} ` : '') +
-              (event.error ? `: ${event.error}` : ''),
-          ),
+/** @type {XMLHttpRequest | undefined} */
+var pendingXhr;
+/** @type {boolean} */
+var hasPendingUpdate = false;
+
+var sendRequest = function sendRequest() {
+  if (compiling.size === 0) {
+    hasPendingUpdate = false;
+    return;
+  }
+
+  var modules = Array.from(compiling);
+  var data = modules.join('\n');
+
+  var xhr = new XMLHttpRequest();
+  pendingXhr = xhr;
+  xhr.open('POST', urlBase, true);
+  // text/plain Content-Type is simple request header
+  xhr.setRequestHeader('Content-Type', 'text/plain');
+
+  xhr.onreadystatechange = function () {
+    if (xhr.readyState === 4) {
+      pendingXhr = undefined;
+      if (xhr.status < 200 || xhr.status >= 300) {
+        var error = new Error(
+          'Problem communicating active modules to the server: HTTP ' +
+            xhr.status,
         );
-      });
-    };
-  } else {
-    activeEventSource = undefined;
+        errorHandlers.forEach(function (onError) {
+          onError(error);
+        });
+      }
+      if (hasPendingUpdate) {
+        hasPendingUpdate = false;
+        sendRequest();
+      }
+    }
+  };
+
+  xhr.onerror = function () {
+    pendingXhr = undefined;
+    var error = new Error('Problem communicating active modules to the server');
+    errorHandlers.forEach(function (onError) {
+      onError(error);
+    });
+  };
+
+  xhr.send(data);
+};
+
+var sendActiveRequest = function sendActiveRequest() {
+  hasPendingUpdate = true;
+
+  // If no request is pending, start one
+  if (!pendingXhr) {
+    hasPendingUpdate = false;
+    sendRequest();
   }
 };
 
@@ -55,7 +81,7 @@ exports.activate = function (options) {
 
   if (!compiling.has(data)) {
     compiling.add(data);
-    updateEventSource();
+    sendActiveRequest();
   }
 
   if (!active && !module.hot) {
@@ -67,6 +93,6 @@ exports.activate = function (options) {
   return function () {
     errorHandlers.delete(onError);
     compiling.delete(data);
-    updateEventSource();
+    sendActiveRequest();
   };
 };

packages/rspack/src/builtin-plugin/lazy-compilation/middleware.ts

@@ -43,14 +43,6 @@ const DEPRECATED_LAZY_COMPILATION_OPTIONS_WARN =
 const REPEAT_LAZY_COMPILATION_OPTIONS_WARN =
   'Both top-level `lazyCompilation` and `experiments.lazyCompilation` options are set. The top-level `lazyCompilation` configuration will take precedence.';
 
-/**
- * Create a middleware that handles lazy compilation requests from the client.
- * This function returns an Express-style middleware that listens for
- * requests triggered by lazy compilation in the dev server client,
- * then invokes the Rspack compiler to compile modules on demand.
- * Use this middleware when integrating lazy compilation into a
- * custom development server instead of relying on the built-in server.
- */
 export const lazyCompilationMiddleware = (
   compiler: Compiler | MultiCompiler,
 ): MiddlewareHandler => {
@@ -88,7 +80,6 @@ export const lazyCompilationMiddleware = (
       }
 
       const options = {
-        // TODO: remove this when experiments.lazyCompilation is removed
         ...c.options.experiments.lazyCompilation,
         ...c.options.lazyCompilation,
       };
@@ -139,7 +130,6 @@ export const lazyCompilationMiddleware = (
   const activeModules: Set<string> = new Set();
 
   const options = {
-    // TODO: remove this when experiments.lazyCompilation is removed
     ...compiler.options.experiments.lazyCompilation,
     ...compiler.options.lazyCompilation,
   };
@@ -173,24 +163,67 @@ function applyPlugin(
   plugin.apply(compiler);
 }
 
-// used for reuse code, do not export this
+function readModuleIdsFromBody(
+  req: IncomingMessage & { body?: unknown },
+): Promise<string[]> {
+  // If body is already parsed by another middleware, use it directly
+  if (req.body !== undefined) {
+    if (Array.isArray(req.body)) {
+      return Promise.resolve(req.body);
+    }
+    if (typeof req.body === 'string') {
+      return Promise.resolve(req.body.split('\n').filter(Boolean));
+    }
+    throw new Error('Invalid body type');
+  }
+
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on('data', (chunk: Buffer) => {
+      chunks.push(chunk);
+    });
+    req.on('end', () => {
+      // Concatenate all chunks and decode as UTF-8 to handle multi-byte characters correctly
+      const body = Buffer.concat(chunks).toString('utf8');
+      resolve(body.split('\n').filter(Boolean));
+    });
+    req.on('error', reject);
+  });
+}
+
 const lazyCompilationMiddlewareInternal = (
   compiler: Compiler | MultiCompiler,
   activeModules: Set<string>,
   lazyCompilationPrefix: string,
 ): MiddlewareHandler => {
   const logger = compiler.getInfrastructureLogger('LazyCompilation');
 
-  return (req: IncomingMessage, res: ServerResponse, next?: () => void) => {
+  return async (
+    req: IncomingMessage,
+    res: ServerResponse,
+    next?: () => void,
+  ) => {
     if (!req.url?.startsWith(lazyCompilationPrefix)) {
-      // only handle requests that are come from lazyCompilation
       return next?.();
     }
 
-    const modules = req.url
-      .slice(lazyCompilationPrefix.length)
-      .split('@')
-      .map(decodeURIComponent);
+    let modules: string[] = [];
+    if (req.method === 'POST') {
+      try {
+        modules = await readModuleIdsFromBody(req);
+      } catch (err) {
+        logger.error('Failed to parse request body: ' + err);
+        res.writeHead(400);
+        res.end('Bad Request');
+        return;
+      }
+    } else {
+      modules = req.url
+        .slice(lazyCompilationPrefix.length)
+        .split('@')
+        .map(decodeURIComponent);
+    }
+
     req.socket.setNoDelay(true);
 
     res.setHeader('content-type', 'text/event-stream');
@@ -210,5 +243,8 @@ const lazyCompilationMiddlewareInternal = (
     if (moduleActivated.length && compiler.watching) {
       compiler.watching.invalidate();
     }
+    if (req.method === 'POST') {
+      res.end();
+    }
   };
 };

tests/e2e/cases/lazy-compilation/active-lots-modules/index.test.ts

@@ -0,0 +1,6 @@
+import { expect, test } from '@/fixtures';
+
+test('should update style', async ({ page }) => {
+  const body = await page.locator('body');
+  await expect(body).toContainText('All Modules Loaded');
+});

tests/e2e/cases/lazy-compilation/active-lots-modules/rspack.config.js

@@ -0,0 +1,43 @@
+const { rspack } = require('@rspack/core');
+
+/*
+Construct a project with lots of virtual files with very long file names
+And `virtual_index.js` dynamically imports all the long file name files.
+src/
+├── virtual_index.js
+├── virtual_with_a_very_long_file_name_number_....._0.js
+...
+└── virtual_with_a_very_long_file_name_number_....._19.js
+*/
+let lotsLongFileNameVirtualFiles = {};
+let longStr = new Array(1024).fill('a').join('');
+for (let i = 0; i < 20; i++) {
+  lotsLongFileNameVirtualFiles[
+    `src/virtual_with_a_very_long_file_name_number_${longStr}_${i}.js`
+  ] = `"dynamic_imported"`;
+}
+let allFiles = Object.keys(lotsLongFileNameVirtualFiles);
+lotsLongFileNameVirtualFiles['src/virtual_index.js'] = `
+  Promise.all([
+    ${allFiles.map((file) => `import('./${file.slice(3)}')\n`).join(',')}
+  ]).then(()=> document.body.innerHTML = 'All Modules Loaded');
+`;
+
+/** @type { import('@rspack/core').RspackOptions } */
+module.exports = {
+  context: __dirname,
+  entry: './src/virtual_index.js',
+  mode: 'development',
+  lazyCompilation: true,
+  devServer: {
+    hot: true,
+    port: 5678,
+  },
+  plugins: [
+    new rspack.HtmlRspackPlugin(),
+    new rspack.experiments.VirtualModulesPlugin(lotsLongFileNameVirtualFiles),
+  ],
+  experiments: {
+    useInputFileSystem: [/virtual/],
+  },
+};

tests/e2e/cases/lazy-compilation/cross-origin/README.md

@@ -0,0 +1,46 @@
+# Cross-Origin Lazy Compilation Test
+
+This test verifies that lazy compilation works correctly
+when the lazy compilation server runs on a different origin (port) than the frontend dev server.
+
+## Architecture
+
+```
++----------------------+    +----------------------+    +-----------------------------+
+|                      |    |                      |    |                             |
+|      Browser         |    |      Dev Server      |    |   Lazy Compilation Server   |
+|      (Frontend)      |    |      (Port: 8500)    |    |        (Port: 8600)          |
+|                      |    |                      |    |                             |
++----------+-----------+    +----------+-----------+    +--------------+--------------+
+           |                         |                                 |
+           |  1. Load page           |                                 |
+           |  GET http://localhost:8500                                |
+           +------------------------->                                 |
+           |                         |                                 |
+           |  2. Click button -> dynamic import()                      |
+           |                         |                                 |
+           |  3. Cross-origin POST request                             |
+           |     POST http://127.0.0.1:8600/lazy-...                   |
+           |     Content-Type: text/plain                              |
+           |     Body: "module-id-1\nmodule-id-2"                      |
+           +---------------------------------------------------------->
+           |                         |                                 |
+           |  4. Response (SSE or empty for POST)                      |
+           <----------------------------------------------------------+
+           |                         |                                 |
+           |  5. Webpack invalidate & rebuild                          |
+           |                         |                                 |
+           |  6. Load compiled chunk                                   |
+           v                         |                                 |
++----------------------+             |                                 |
+|      Component       |             |                                 |
+|      Rendered        |             |                                 |
++----------------------+             |                                 |
+```
+
+## Key Points
+
+1. **Two Separate Servers**: Frontend runs on port 8500, lazy compilation on port 8600
+2. **Cross-Origin Request**: Browser sends POST request to a different origin
+3. **Simple Request**: Uses `Content-Type: text/plain` to avoid CORS preflight
+4. **XMLHttpRequest**: Uses XHR instead of fetch for better browser compatibility