ci: fix dynamic-import response truncation flake (proxy transport race) + socket instrumentation

.github/workflows/ci-test-e2e.yml

@@ -320,6 +320,10 @@ jobs:
         if: failure()
         run: docker compose -f docker-compose-ci.yml logs mongo
 
+      - name: Show traefik logs if E2E test failed
+        if: failure()
+        run: docker compose -f docker-compose-ci.yml logs traefik
+
       - name: Store coverage
         if: inputs.coverage == matrix.mongodb-version
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

apps/meteor/server/startup/httpSocketTimeout.ts

@@ -0,0 +1,17 @@
+import { WebApp } from 'meteor/webapp';
+
+// meteor's webapp arms a 5s idle-socket timeout (and re-arms it after every
+// response), which kills keep-alive conns a fronting proxy may be about to
+// reuse — the root of the CI dynamic-import truncation flake (PR #41399).
+// Overriding it is opt-in: only CI sets HTTP_SOCKET_TIMEOUT_MS, pointing it
+// above traefik's idleConnTimeout so the proxy always closes conns first.
+const timeout = parseInt(process.env.HTTP_SOCKET_TIMEOUT_MS ?? '', 10);
+if (timeout > 0) {
+	const { httpServer, _timeoutAdjustmentRequestCallback } = WebApp as typeof WebApp & {
+		_timeoutAdjustmentRequestCallback: (req: unknown, res: unknown) => void;
+	};
+	httpServer.removeListener('request', _timeoutAdjustmentRequestCallback);
+	httpServer.setTimeout(timeout);
+	httpServer.keepAliveTimeout = timeout;
+	httpServer.headersTimeout = timeout + 1000;
+}

apps/meteor/server/startup/index.ts

@@ -7,6 +7,7 @@ import './serverRunning';
 import './coreApps';
 import { generateFederationKeys } from './generateKeys';
 import './presenceTroubleshoot';
+import './httpSocketTimeout';
 import '../hooks';
 import '../lib/rooms/roomTypes';
 import '../lib/settingsRegenerator';

apps/meteor/tests/end-to-end/api/http-response-truncation.ts

@@ -0,0 +1,86 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import { apiUrl } from '../../data/api-data';
+
+// Regression test for the dynamic-import response truncation flake (PR #41399).
+//
+// Through traefik with `serverstransport.maxidleconnsperhost=-1` (upstream
+// keep-alive disabled), Go's transport rarely closes its upstream connection
+// while the ReverseProxy is still copying a chunked gzip response body
+// ("use of closed network connection"), truncating the response mid-stream:
+// the browser gets 200 headers and then ERR_INCOMPLETE_CHUNKED_ENCODING, and
+// the meteor module loader hangs the page. Rate is ~1.6e-4 per request under
+// load, so this test replays the exact request shape (real dynamic-import
+// bodies captured from a failing CI run — the fatal one is the same
+// AppLayoutThemeWrapper.tsx request seen in every reproduction) at high
+// volume and concurrency: red on the broken proxy config, green once
+// upstream keep-alive is restored.
+//
+// The bodies are build-dependent fixtures: if the app tree renames these
+// modules the responses shrink below the gzip threshold and lose the
+// chunked+gzip shape the race needs, so the test asserts that shape first.
+
+const FETCH_URL = `${apiUrl}/__meteor__/dynamic-import/fetch`;
+
+const BODIES = [
+	'{"client":{"components":{"AppLayoutThemeWrapper.tsx":1}}}',
+	'{"client":{"meteor":{"login":{"index.ts":1,"cas.ts":1,"crowd.ts":1,"facebook.ts":1,"oauth.ts":1,"LoginCancelledError.ts":1,"google.ts":1,"ldap.ts":1,"meteorDeveloperAccount.ts":1,"password.ts":1,"saml.ts":1,"twitter.ts":1}},"lib":{"2fa":{"overrideLoginMethod.ts":1},"wrapRequestCredentialFn.ts":1,"loginServices.ts":1}},"node_modules":{"@rocket.chat":{"string-helpers":{"package.json":1,"dist":{"esm":{"index.js":1}}}}}}',
+];
+
+const TOTAL_REQUESTS = 30000;
+const CONCURRENCY = 64;
+
+const fetchModuleTree = async (body: string): Promise<{ error?: string }> => {
+	try {
+		const response = await fetch(FETCH_URL, {
+			method: 'POST',
+			headers: { 'Content-Type': 'text/plain;charset=UTF-8' },
+			body,
+		});
+		if (response.status !== 200) {
+			return { error: `unexpected status ${response.status}` };
+		}
+		await response.text();
+		return {};
+	} catch (error) {
+		return { error: `${error} (cause: ${(error as { cause?: unknown }).cause})` };
+	}
+};
+
+describe('dynamic-import response delivery through the proxy', () => {
+	it('should keep the chunked gzip response shape the race depends on', async () => {
+		const response = await fetch(FETCH_URL, {
+			method: 'POST',
+			headers: { 'Content-Type': 'text/plain;charset=UTF-8' },
+			body: BODIES[1],
+		});
+		const text = await response.text();
+		expect(response.status).to.equal(200);
+		expect(
+			text.length,
+			'fixture went stale: response too small to be gzip-chunked, update BODIES from a fresh browser trace',
+		).to.be.greaterThan(10000);
+	});
+
+	it(`should deliver ${TOTAL_REQUESTS} concurrent dynamic-import responses without a single truncation`, async function () {
+		this.timeout(10 * 60 * 1000);
+
+		const failures: string[] = [];
+		let issued = 0;
+
+		const worker = async (): Promise<void> => {
+			while (issued < TOTAL_REQUESTS && failures.length === 0) {
+				const current = issued++;
+				const { error } = await fetchModuleTree(BODIES[current % BODIES.length]);
+				if (error) {
+					failures.push(`request #${current}: ${error}`);
+				}
+			}
+		};
+
+		await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+
+		expect(failures).to.deep.equal([]);
+	});
+});

docker-compose-ci.yml

@@ -29,6 +29,9 @@ services:
       - Federation_Service_Enabled=true
       - 'Federation_Service_Domain=rc.host'
       - HEAP_USAGE_PERCENT=99
+      # keep server sockets alive longer than traefik's 90s idleConnTimeout so
+      # the proxy always closes pooled conns first and never reuses a dead one
+      - HTTP_SOCKET_TIMEOUT_MS=120000
     depends_on:
       - traefik
       - mongo
@@ -37,7 +40,17 @@ services:
       traefik.http.services.rocketchat.loadbalancer.server.port: 3000
       traefik.http.routers.rocketchat.service: rocketchat
       traefik.http.routers.rocketchat.rule: PathPrefix(`/`)
+      traefik.http.routers.rocketchat.middlewares: test-retry
       traefik.http.middlewares.test-retry.retry.attempts: 4
+      # dedicated route for dynamic-import: buffering coalesces the tiny POST
+      # into a single upstream write, closing the go-transport early-response
+      # race window (response beating the request-write bookkeeping kills the
+      # conn mid body copy); scoped here because suite-wide buffering broke
+      # 404/CORS response semantics
+      traefik.http.routers.rocketchat-dynimport.rule: PathPrefix(`/__meteor__/dynamic-import`)
+      traefik.http.routers.rocketchat-dynimport.service: rocketchat
+      traefik.http.routers.rocketchat-dynimport.middlewares: test-buffer,test-retry
+      traefik.http.middlewares.test-buffer.buffering.retryExpression: IsNetworkError() && Attempts() < 4
     healthcheck:
       interval: 2s
       timeout: 5s
@@ -217,7 +230,6 @@ services:
     image: traefik:v3.6.6
     command:
       - --providers.docker=true
-      - '--serverstransport.maxidleconnsperhost=-1'
     ports:
       - 3000:80
     volumes: