diff --git a/.github/workflows/ci-test-e2e.yml b/.github/workflows/ci-test-e2e.yml index f89fa0ac3ee9e..090a6f5a8ed5a 100644 --- a/.github/workflows/ci-test-e2e.yml +++ b/.github/workflows/ci-test-e2e.yml @@ -320,6 +320,10 @@ jobs: if: failure() run: docker compose -f docker-compose-ci.yml logs mongo + - name: Show traefik logs if E2E test failed + if: failure() + run: docker compose -f docker-compose-ci.yml logs traefik + - name: Store coverage if: inputs.coverage == matrix.mongodb-version uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/apps/meteor/server/startup/httpSocketTimeout.ts b/apps/meteor/server/startup/httpSocketTimeout.ts new file mode 100644 index 0000000000000..275f9bc10fd1d --- /dev/null +++ b/apps/meteor/server/startup/httpSocketTimeout.ts @@ -0,0 +1,17 @@ +import { WebApp } from 'meteor/webapp'; + +// meteor's webapp arms a 5s idle-socket timeout (and re-arms it after every +// response), which kills keep-alive conns a fronting proxy may be about to +// reuse — the root of the CI dynamic-import truncation flake (PR #41399). +// Overriding it is opt-in: only CI sets HTTP_SOCKET_TIMEOUT_MS, pointing it +// above traefik's idleConnTimeout so the proxy always closes conns first. +const timeout = parseInt(process.env.HTTP_SOCKET_TIMEOUT_MS ?? '', 10); +if (timeout > 0) { + const { httpServer, _timeoutAdjustmentRequestCallback } = WebApp as typeof WebApp & { + _timeoutAdjustmentRequestCallback: (req: unknown, res: unknown) => void; + }; + httpServer.removeListener('request', _timeoutAdjustmentRequestCallback); + httpServer.setTimeout(timeout); + httpServer.keepAliveTimeout = timeout; + httpServer.headersTimeout = timeout + 1000; +} diff --git a/apps/meteor/server/startup/index.ts b/apps/meteor/server/startup/index.ts index d808423188074..b5fc18d36a9b5 100644 --- a/apps/meteor/server/startup/index.ts +++ b/apps/meteor/server/startup/index.ts @@ -7,6 +7,7 @@ import './serverRunning'; import './coreApps'; import { generateFederationKeys } from './generateKeys'; import './presenceTroubleshoot'; +import './httpSocketTimeout'; import '../hooks'; import '../lib/rooms/roomTypes'; import '../lib/settingsRegenerator'; diff --git a/apps/meteor/tests/end-to-end/api/http-response-truncation.ts b/apps/meteor/tests/end-to-end/api/http-response-truncation.ts new file mode 100644 index 0000000000000..1a6a07a0cb9f9 --- /dev/null +++ b/apps/meteor/tests/end-to-end/api/http-response-truncation.ts @@ -0,0 +1,86 @@ +import { expect } from 'chai'; +import { describe, it } from 'mocha'; + +import { apiUrl } from '../../data/api-data'; + +// Regression test for the dynamic-import response truncation flake (PR #41399). +// +// Through traefik with `serverstransport.maxidleconnsperhost=-1` (upstream +// keep-alive disabled), Go's transport rarely closes its upstream connection +// while the ReverseProxy is still copying a chunked gzip response body +// ("use of closed network connection"), truncating the response mid-stream: +// the browser gets 200 headers and then ERR_INCOMPLETE_CHUNKED_ENCODING, and +// the meteor module loader hangs the page. Rate is ~1.6e-4 per request under +// load, so this test replays the exact request shape (real dynamic-import +// bodies captured from a failing CI run — the fatal one is the same +// AppLayoutThemeWrapper.tsx request seen in every reproduction) at high +// volume and concurrency: red on the broken proxy config, green once +// upstream keep-alive is restored. +// +// The bodies are build-dependent fixtures: if the app tree renames these +// modules the responses shrink below the gzip threshold and lose the +// chunked+gzip shape the race needs, so the test asserts that shape first. + +const FETCH_URL = `${apiUrl}/__meteor__/dynamic-import/fetch`; + +const BODIES = [ + '{"client":{"components":{"AppLayoutThemeWrapper.tsx":1}}}', + '{"client":{"meteor":{"login":{"index.ts":1,"cas.ts":1,"crowd.ts":1,"facebook.ts":1,"oauth.ts":1,"LoginCancelledError.ts":1,"google.ts":1,"ldap.ts":1,"meteorDeveloperAccount.ts":1,"password.ts":1,"saml.ts":1,"twitter.ts":1}},"lib":{"2fa":{"overrideLoginMethod.ts":1},"wrapRequestCredentialFn.ts":1,"loginServices.ts":1}},"node_modules":{"@rocket.chat":{"string-helpers":{"package.json":1,"dist":{"esm":{"index.js":1}}}}}}', +]; + +const TOTAL_REQUESTS = 30000; +const CONCURRENCY = 64; + +const fetchModuleTree = async (body: string): Promise<{ error?: string }> => { + try { + const response = await fetch(FETCH_URL, { + method: 'POST', + headers: { 'Content-Type': 'text/plain;charset=UTF-8' }, + body, + }); + if (response.status !== 200) { + return { error: `unexpected status ${response.status}` }; + } + await response.text(); + return {}; + } catch (error) { + return { error: `${error} (cause: ${(error as { cause?: unknown }).cause})` }; + } +}; + +describe('dynamic-import response delivery through the proxy', () => { + it('should keep the chunked gzip response shape the race depends on', async () => { + const response = await fetch(FETCH_URL, { + method: 'POST', + headers: { 'Content-Type': 'text/plain;charset=UTF-8' }, + body: BODIES[1], + }); + const text = await response.text(); + expect(response.status).to.equal(200); + expect( + text.length, + 'fixture went stale: response too small to be gzip-chunked, update BODIES from a fresh browser trace', + ).to.be.greaterThan(10000); + }); + + it(`should deliver ${TOTAL_REQUESTS} concurrent dynamic-import responses without a single truncation`, async function () { + this.timeout(10 * 60 * 1000); + + const failures: string[] = []; + let issued = 0; + + const worker = async (): Promise => { + while (issued < TOTAL_REQUESTS && failures.length === 0) { + const current = issued++; + const { error } = await fetchModuleTree(BODIES[current % BODIES.length]); + if (error) { + failures.push(`request #${current}: ${error}`); + } + } + }; + + await Promise.all(Array.from({ length: CONCURRENCY }, worker)); + + expect(failures).to.deep.equal([]); + }); +}); diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index c2c23488913f6..cef50ced7cbf4 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -29,6 +29,9 @@ services: - Federation_Service_Enabled=true - 'Federation_Service_Domain=rc.host' - HEAP_USAGE_PERCENT=99 + # keep server sockets alive longer than traefik's 90s idleConnTimeout so + # the proxy always closes pooled conns first and never reuses a dead one + - HTTP_SOCKET_TIMEOUT_MS=120000 depends_on: - traefik - mongo @@ -37,7 +40,17 @@ services: traefik.http.services.rocketchat.loadbalancer.server.port: 3000 traefik.http.routers.rocketchat.service: rocketchat traefik.http.routers.rocketchat.rule: PathPrefix(`/`) + traefik.http.routers.rocketchat.middlewares: test-retry traefik.http.middlewares.test-retry.retry.attempts: 4 + # dedicated route for dynamic-import: buffering coalesces the tiny POST + # into a single upstream write, closing the go-transport early-response + # race window (response beating the request-write bookkeeping kills the + # conn mid body copy); scoped here because suite-wide buffering broke + # 404/CORS response semantics + traefik.http.routers.rocketchat-dynimport.rule: PathPrefix(`/__meteor__/dynamic-import`) + traefik.http.routers.rocketchat-dynimport.service: rocketchat + traefik.http.routers.rocketchat-dynimport.middlewares: test-buffer,test-retry + traefik.http.middlewares.test-buffer.buffering.retryExpression: IsNetworkError() && Attempts() < 4 healthcheck: interval: 2s timeout: 5s @@ -217,7 +230,6 @@ services: image: traefik:v3.6.6 command: - --providers.docker=true - - '--serverstransport.maxidleconnsperhost=-1' ports: - 3000:80 volumes: