From 7e5925385be8ac8a32b685a4c30d029ed816c6cc Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:51:17 +0000 Subject: [PATCH 1/3] quic: use Cubic congestion control for the HTTP/3 server and client engines lsquic's default adaptive mode picks BBRv1 whenever a connection's handshake RTT is over 1.5 ms, which any JS work on the loop that ticks the engine produces. BBRv1 can then only ever observe one cwnd per loop iteration as bandwidth, so cwnd never grows in startup and collapses to the 4-packet minimum once startup ends. Pin both engines to Cubic, which only reacts to loss, and add tests that measure how much a response body and a streamed request body move while the serving loop is busy. --- packages/bun-usockets/src/quic.c | 16 ++++ test/js/bun/http/serve-http3.test.ts | 124 +++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/packages/bun-usockets/src/quic.c b/packages/bun-usockets/src/quic.c index 8926c7cfbd8a..42077949d7f6 100644 --- a/packages/bun-usockets/src/quic.c +++ b/packages/bun-usockets/src/quic.c @@ -690,6 +690,20 @@ static void us_quic_prepare_ssl_ctx(SSL_CTX *ssl) { SSL_CTX_set_early_data_enabled(ssl, 0); } +/* Both engines are only ticked from an event loop (see "process driver" + * above), so every RTT lsquic measures includes whatever the loop ran between + * two ticks. lsquic's default "adaptive" CC (es_cc_algo 3) makes a permanent + * choice from a connection's handshake RTT, BBRv1 above 1.5 ms, which any JS + * work during the handshake exceeds even on loopback. BBRv1 then models loop + * latency as the path: the only bandwidth it can observe is one cwnd per loop + * iteration, so cwnd never grows in startup and, once startup ends, shrinks to + * the 4-packet minimum; a busy server then moves ~7 KB per loop iteration. + * Cubic only reacts to loss, which a slow loop doesn't produce, and is what + * adaptive mode picks on a quiet loop anyway. */ +static void us_quic_use_cubic(struct lsquic_engine_settings *settings) { + settings->es_cc_algo = 1; +} + us_quic_socket_context_t *us_create_quic_socket_context( struct us_loop_t *loop, struct us_bun_socket_context_options_t options, unsigned int ext_size, unsigned int idle_timeout_s) @@ -706,6 +720,7 @@ us_quic_socket_context_t *us_create_quic_socket_context( ctx->ssl_ctx = ssl; lsquic_engine_init_settings(&ctx->settings, LSENG_HTTP_SERVER); + us_quic_use_cubic(&ctx->settings); ctx->settings.es_versions = LSQUIC_DF_VERSIONS & LSQUIC_IETF_VERSIONS; ctx->settings.es_ecn = 0; /* QPACK can expand small dynamic-table refs into large header lists; cap @@ -1170,6 +1185,7 @@ us_quic_socket_context_t *us_create_quic_client_context( ctx->stream_ext_size = stream_ext_size; lsquic_engine_init_settings(&ctx->settings, LSENG_HTTP); + us_quic_use_cubic(&ctx->settings); ctx->settings.es_versions = (1u << LSQVER_I001); ctx->settings.es_ecn = 0; ctx->settings.es_max_header_list_size = 64 * 1024; diff --git a/test/js/bun/http/serve-http3.test.ts b/test/js/bun/http/serve-http3.test.ts index 2f63384c3477..305add70564d 100644 --- a/test/js/bun/http/serve-http3.test.ts +++ b/test/js/bun/http/serve-http3.test.ts @@ -1301,3 +1301,127 @@ describe("Bun.serve HTTP/3 production", () => { // (HttpContext.h / Http3Context.h call writeContinue before routing); a // curl --expect100-timeout assertion was flaky enough to drop here. }); + +// Both QUIC engines (Bun.serve's and the fetch client's) are only ticked from +// an event loop, so every RTT lsquic measures includes whatever that loop ran +// in between. quic.c pins the congestion controller to Cubic: lsquic's default +// "adaptive" mode settles on BBRv1 for good as soon as a connection's early RTT +// samples exceed 1.5 ms, and BBRv1 then models loop latency as the path. Its +// window target is bandwidth x min RTT, where the "bandwidth" it can observe +// on a busy loop is one window per loop iteration and the min RTT is whatever +// the quickest moment of the connection looked like, so once it leaves startup +// the window shrinks round after round until it sits at the 4-packet minimum. +// Cubic only reacts to loss, which a slow loop does not produce. +// +// The fixture counts its own event loop iterations and paces them with +// Bun.sleepSync: 5 ms per iteration while the connection is set up (so the +// handshake RTTs that adaptive mode decides on are well over 1.5 ms), idle for +// a moment once the transfer starts (so the controller's min RTT is the real +// loopback RTT, as it would be for any connection that saw one quick round +// trip), then 12 ms per iteration for ITERATIONS iterations. Only the second +// half of those is asserted on, which leaves either controller ample rounds to +// settle. Over those 60 iterations Cubic moves several MiB (it is bounded by +// the receiver's UDP buffer: 3.5-7 MiB with Linux's 208 KiB default on a debug +// build), a collapsed BBRv1 window moves ~7 KiB per iteration, 0.4-0.5 MiB. +describe("Bun.serve HTTP/3 congestion control", () => { + const ITERATIONS = 120; + // Larger than any window either controller reaches here, so every tick is + // limited by the congestion window, not by how much the application had + // queued. (Application-limited rounds never let BBRv1 leave startup, which + // is why a response made of small chunks does not show the collapse.) + const CHUNK_SIZE = 512 * 1024; + const MIN_SECOND_HALF_BYTES = 1024 * 1024; + + const script = ` + const tls = ${JSON.stringify(tls)}; + const ITERATIONS = ${ITERATIONS}; + const CHUNK = Buffer.alloc(${CHUNK_SIZE}, "cwnd"); + let busyMs = 5; + let iterations = 0; + let windowStart = Infinity; + (function spin() { + iterations++; + if (iterations >= windowStart + ITERATIONS) busyMs = 0; + if (busyMs) Bun.sleepSync(busyMs); + setImmediate(spin); + })(); + function startTransfer() { + busyMs = 0; + setTimeout(() => { windowStart = iterations; busyMs = 12; }, 20); + } + // Iterations since the busy window opened; negative before that. + const elapsed = () => iterations - windowStart; + const inSecondHalf = () => elapsed() >= ITERATIONS / 2; + let download = { chunks: 0, secondHalfChunks: 0 }; + const server = Bun.serve({ + port: 0, tls, http3: true, http1: false, + async fetch(req) { + const { pathname } = new URL(req.url); + if (pathname === "/download") { + startTransfer(); + download = { chunks: 0, secondHalfChunks: 0 }; + return new Response(new ReadableStream({ + // pull() runs once the previous chunk has been handed to lsquic in + // full, so chunks pulled during the second half of the window are + // the chunks the connection actually moved during it (+-1). + pull(ctrl) { + if (elapsed() >= ITERATIONS) return ctrl.close(); + download.chunks++; + if (inSecondHalf()) download.secondHalfChunks++; + ctrl.enqueue(CHUNK); + }, + })); + } + if (pathname === "/download/stats") return Response.json(download); + if (pathname === "/upload") { + startTransfer(); + let total = 0, secondHalf = 0; + for await (const chunk of req.body) { + if (elapsed() >= ITERATIONS) break; + total += chunk.length; + if (inSecondHalf()) secondHalf += chunk.length; + } + return Response.json({ total, secondHalf }); + } + return new Response("not found", { status: 404 }); + }, + }); + console.error("PORT=" + server.port); + process.stdin.on("data", () => {}); + ${STOP_ON_STDIN_END} + `; + + test("a response body keeps flowing while the server's event loop is busy", async () => { + await withCustomServer(script, async port => { + const body = await fetchH3(port, "/download").then(r => r.bytes()); + const { chunks, secondHalfChunks } = (await fetchH3(port, "/download/stats").then(r => r.json())) as { + chunks: number; + secondHalfChunks: number; + }; + expect(body.length).toBe(chunks * CHUNK_SIZE); + expect(secondHalfChunks * CHUNK_SIZE).toBeGreaterThanOrEqual(MIN_SECOND_HALF_BYTES); + }); + }); + + // The fetch client's engine sees the same thing from the other side: the + // server's loop delays every ACK, so its RTT samples are inflated the same way. + test("a streamed request body keeps flowing while the server's event loop is busy", async () => { + await withCustomServer(script, async port => { + const chunk = Buffer.alloc(CHUNK_SIZE, "cwnd"); + let responded = false; + const res = await fetchH3(port, "/upload", { + method: "POST", + body: new ReadableStream({ + pull(ctrl) { + if (responded) ctrl.close(); + else ctrl.enqueue(chunk); + }, + }), + }); + responded = true; + const { total, secondHalf } = (await res.json()) as { total: number; secondHalf: number }; + expect(total).toBeGreaterThanOrEqual(secondHalf); + expect(secondHalf).toBeGreaterThanOrEqual(MIN_SECOND_HALF_BYTES); + }); + }); +}); From d4337355d4909b4d9aa11625f18d2ceb705240dd Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:59:00 +0000 Subject: [PATCH 2/3] lsquic: widen Cubic's window arithmetic so it does not wrap on Windows Cubic keeps its windows in unsigned long, which is 32 bits on Windows, and its pacing rate is cwnd * 1000000 / srtt computed in that type, so the product wrapped for every window and Cubic connections on Windows were paced at an essentially random rate. The two loss-reduction products wrap the same way once the window passes ~4.6 MB. With both engines pinned to Cubic every HTTP/3 connection on Windows would have hit the pacing one. --- patches/lsquic/cubic-llp64-overflow.patch | 35 +++++++++++++++++++++++ scripts/build/deps/lsquic.ts | 6 ++++ 2 files changed, 41 insertions(+) create mode 100644 patches/lsquic/cubic-llp64-overflow.patch diff --git a/patches/lsquic/cubic-llp64-overflow.patch b/patches/lsquic/cubic-llp64-overflow.patch new file mode 100644 index 000000000000..4b396cbf81d6 --- /dev/null +++ b/patches/lsquic/cubic-llp64-overflow.patch @@ -0,0 +1,35 @@ +--- a/src/liblsquic/lsquic_cubic.c ++++ b/src/liblsquic/lsquic_cubic.c +@@ -204,11 +204,14 @@ lsquic_cubic_loss (void *cong_ctl) + struct lsquic_cubic *const cubic = cong_ctl; + LSQ_DEBUG("%s(cubic)", __func__); + cubic->cu_epoch_start = 0; ++ /* bun: the window fields are unsigned long, which is 32 bits on Windows ++ * (LLP64), so widen before multiplying; see lsquic_cubic_pacing_rate. */ + if (FAST_CONVERGENCE && cubic->cu_cwnd < cubic->cu_last_max_cwnd) +- cubic->cu_last_max_cwnd = cubic->cu_cwnd * TWO_MINUS_BETA_OVER_TWO / 1024; ++ cubic->cu_last_max_cwnd = (uint64_t) cubic->cu_cwnd ++ * TWO_MINUS_BETA_OVER_TWO / 1024; + else + cubic->cu_last_max_cwnd = cubic->cu_cwnd; +- cubic->cu_cwnd = cubic->cu_cwnd * ONE_MINUS_BETA / 1024; ++ cubic->cu_cwnd = (uint64_t) cubic->cu_cwnd * ONE_MINUS_BETA / 1024; + cubic->cu_tcp_cwnd = cubic->cu_cwnd; + cubic->cu_ssthresh = cubic->cu_cwnd; + LSQ_INFO("loss detected, last_max_cwnd: %lu, cwnd: %lu", +@@ -266,7 +269,14 @@ lsquic_cubic_pacing_rate (void *cong_ctl, int in_recovery) + srtt = lsquic_rtt_stats_get_srtt(cubic->cu_rtt_stats); + if (srtt == 0) + srtt = 50000; +- bandwidth = cubic->cu_cwnd * 1000000 / srtt; ++ /* bun: cu_cwnd is unsigned long, 32 bits on Windows (LLP64), so this ++ * product wrapped for every window above 4 KB and the pacing rate came ++ * out as (cwnd * 10^6 mod 2^32) / srtt: a Cubic connection on Windows ++ * was paced at a more or less random rate, usually a small fraction of ++ * cwnd/srtt. Reported upstream as a Windows throughput problem that went ++ * away with pacing disabled or BBR selected: ++ * https://github.com/litespeedtech/lsquic/issues/391 */ ++ bandwidth = (uint64_t) cubic->cu_cwnd * 1000000 / srtt; + if (in_slow_start(cubic)) + pacing_rate = bandwidth * 2; + else if (in_recovery) diff --git a/scripts/build/deps/lsquic.ts b/scripts/build/deps/lsquic.ts index 177bb94b2917..c02068a0d9c8 100644 --- a/scripts/build/deps/lsquic.ts +++ b/scripts/build/deps/lsquic.ts @@ -128,6 +128,12 @@ export const lsquic: Dependency = { // never be encrypted and the peer idled out instead of learning of the // close. Select the PNS by handshake progress, as ngtcp2 does. "patches/lsquic/connection-close-pns.patch", + // Cubic keeps its windows in unsigned long, 32 bits on Windows, and + // multiplied them unwidened: the pacing rate (cwnd * 10^6 / srtt) wrapped + // for every window, so Cubic connections on Windows were paced at a + // garbage rate (litespeedtech/lsquic#391). quic.c pins both engines to + // Cubic, so every HTTP/3 connection on Windows would hit it. + "patches/lsquic/cubic-llp64-overflow.patch", ], fetchDeps: ["zlib", "lshpack", "lsqpack", "boringssl"], From 47b966e646bd5d8643c46893d8e8d6bf48333ff8 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:59:00 +0000 Subject: [PATCH 3/3] test: make the h3 congestion control tests measure a settled connection Prime both engines with two idle round trips before the busy window so the controller under test has a real loopback min RTT (lsquic takes no sample for packets sent before the first acknowledgment), open the busy window when the transfer starts instead of after an idle phase that left Cubic loss-capped, and bound the server's count by what the client sent instead of an assertion that held by construction. --- test/js/bun/http/serve-http3.test.ts | 87 ++++++++++++++++++---------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/test/js/bun/http/serve-http3.test.ts b/test/js/bun/http/serve-http3.test.ts index 305add70564d..4fdf647d7e64 100644 --- a/test/js/bun/http/serve-http3.test.ts +++ b/test/js/bun/http/serve-http3.test.ts @@ -1305,26 +1305,36 @@ describe("Bun.serve HTTP/3 production", () => { // Both QUIC engines (Bun.serve's and the fetch client's) are only ticked from // an event loop, so every RTT lsquic measures includes whatever that loop ran // in between. quic.c pins the congestion controller to Cubic: lsquic's default -// "adaptive" mode settles on BBRv1 for good as soon as a connection's early RTT -// samples exceed 1.5 ms, and BBRv1 then models loop latency as the path. Its -// window target is bandwidth x min RTT, where the "bandwidth" it can observe -// on a busy loop is one window per loop iteration and the min RTT is whatever -// the quickest moment of the connection looked like, so once it leaves startup -// the window shrinks round after round until it sits at the 4-packet minimum. -// Cubic only reacts to loss, which a slow loop does not produce. +// "adaptive" mode settles on BBRv1 for good when a connection's handshake RTT +// is over 1.5 ms, and BBRv1 then models loop latency as the path. Its window +// target is bandwidth x min RTT, where the only bandwidth it can observe on a +// busy loop is one window per loop iteration and the min RTT is the quickest +// round trip the connection ever saw, so as soon as it leaves startup the +// window drops to the 4-packet minimum and stays there. Cubic only reacts to +// loss, which a slow loop does not produce. // // The fixture counts its own event loop iterations and paces them with -// Bun.sleepSync: 5 ms per iteration while the connection is set up (so the -// handshake RTTs that adaptive mode decides on are well over 1.5 ms), idle for -// a moment once the transfer starts (so the controller's min RTT is the real -// loopback RTT, as it would be for any connection that saw one quick round -// trip), then 12 ms per iteration for ITERATIONS iterations. Only the second -// half of those is asserted on, which leaves either controller ample rounds to -// settle. Over those 60 iterations Cubic moves several MiB (it is bounded by -// the receiver's UDP buffer: 3.5-7 MiB with Linux's 208 KiB default on a debug -// build), a collapsed BBRv1 window moves ~7 KiB per iteration, 0.4-0.5 MiB. +// Bun.sleepSync. It sleeps 5 ms per iteration while the connection is set up, +// so adaptive mode would choose BBRv1; /quiet then stops the sleeping, and the +// round trips made while the loop is idle give both engines a genuine loopback +// RTT sample (any real connection has one from a quiet moment); the transfer +// itself then runs with BUSY_ITERATION_MS of work per iteration for ITERATIONS +// iterations. Only the second half of those is asserted on, which leaves +// either controller ample rounds to settle. +// +// What a healthy connection moves per iteration is bounded by the receiver's +// UDP buffer and by lsquic's pacer, which lets about ten packets plus one +// millisecond's worth through per engine tick, so the numbers are modest but +// far apart: over the measured 60 iterations a collapsed BBRv1 window moves +// 5-6 packets per iteration, ~0.4 MiB in total, while Cubic moves several MiB +// (4-13 MiB on Linux and Windows debug builds). describe("Bun.serve HTTP/3 congestion control", () => { const ITERATIONS = 120; + // Comfortably longer than the idle round trips even on a debug build, where + // they take 1-2 ms: once out of startup, each round multiplies BBRv1's window + // by about 2 x minRTT / iteration, so this has it at the minimum within a + // few rounds. + const BUSY_ITERATION_MS = 12; // Larger than any window either controller reaches here, so every tick is // limited by the congestion window, not by how much the application had // queued. (Application-limited rounds never let BBRv1 leave startup, which @@ -1332,6 +1342,14 @@ describe("Bun.serve HTTP/3 congestion control", () => { const CHUNK_SIZE = 512 * 1024; const MIN_SECOND_HALF_BYTES = 1024 * 1024; + // Two round trips: lsquic takes no bandwidth/RTT sample for a packet sent + // before the connection had its first acknowledgment + // (lsquic_bw_sampler_packet_acked), so the first one only primes the + // sampler and the second is the one the controller measures. + async function quietRoundTrips(port: number) { + for (let i = 0; i < 2; i++) await fetchH3(port, "/quiet").then(r => r.bytes()); + } + const script = ` const tls = ${JSON.stringify(tls)}; const ITERATIONS = ${ITERATIONS}; @@ -1345,27 +1363,32 @@ describe("Bun.serve HTTP/3 congestion control", () => { if (busyMs) Bun.sleepSync(busyMs); setImmediate(spin); })(); - function startTransfer() { - busyMs = 0; - setTimeout(() => { windowStart = iterations; busyMs = 12; }, 20); + function openWindow() { + windowStart = iterations; + busyMs = ${BUSY_ITERATION_MS}; } - // Iterations since the busy window opened; negative before that. - const elapsed = () => iterations - windowStart; - const inSecondHalf = () => elapsed() >= ITERATIONS / 2; + const inSecondHalf = () => iterations - windowStart >= ITERATIONS / 2; + const windowClosed = () => iterations - windowStart >= ITERATIONS; let download = { chunks: 0, secondHalfChunks: 0 }; const server = Bun.serve({ port: 0, tls, http3: true, http1: false, async fetch(req) { const { pathname } = new URL(req.url); + if (pathname === "/quiet") { + busyMs = 0; + // A few packets' worth so the client acknowledges it right away + // instead of waiting out its delayed-ACK timer. + return new Response(Buffer.alloc(4096, "quiet")); + } if (pathname === "/download") { - startTransfer(); + openWindow(); download = { chunks: 0, secondHalfChunks: 0 }; return new Response(new ReadableStream({ // pull() runs once the previous chunk has been handed to lsquic in // full, so chunks pulled during the second half of the window are - // the chunks the connection actually moved during it (+-1). + // the chunks the connection moved during it (+-1). pull(ctrl) { - if (elapsed() >= ITERATIONS) return ctrl.close(); + if (windowClosed()) return ctrl.close(); download.chunks++; if (inSecondHalf()) download.secondHalfChunks++; ctrl.enqueue(CHUNK); @@ -1374,10 +1397,10 @@ describe("Bun.serve HTTP/3 congestion control", () => { } if (pathname === "/download/stats") return Response.json(download); if (pathname === "/upload") { - startTransfer(); + openWindow(); let total = 0, secondHalf = 0; for await (const chunk of req.body) { - if (elapsed() >= ITERATIONS) break; + if (windowClosed()) break; total += chunk.length; if (inSecondHalf()) secondHalf += chunk.length; } @@ -1393,6 +1416,7 @@ describe("Bun.serve HTTP/3 congestion control", () => { test("a response body keeps flowing while the server's event loop is busy", async () => { await withCustomServer(script, async port => { + await quietRoundTrips(port); const body = await fetchH3(port, "/download").then(r => r.bytes()); const { chunks, secondHalfChunks } = (await fetchH3(port, "/download/stats").then(r => r.json())) as { chunks: number; @@ -1407,20 +1431,23 @@ describe("Bun.serve HTTP/3 congestion control", () => { // server's loop delays every ACK, so its RTT samples are inflated the same way. test("a streamed request body keeps flowing while the server's event loop is busy", async () => { await withCustomServer(script, async port => { + await quietRoundTrips(port); const chunk = Buffer.alloc(CHUNK_SIZE, "cwnd"); let responded = false; + let enqueued = 0; const res = await fetchH3(port, "/upload", { method: "POST", body: new ReadableStream({ pull(ctrl) { - if (responded) ctrl.close(); - else ctrl.enqueue(chunk); + if (responded) return ctrl.close(); + enqueued++; + ctrl.enqueue(chunk); }, }), }); responded = true; const { total, secondHalf } = (await res.json()) as { total: number; secondHalf: number }; - expect(total).toBeGreaterThanOrEqual(secondHalf); + expect(total).toBeLessThanOrEqual(enqueued * CHUNK_SIZE); expect(secondHalf).toBeGreaterThanOrEqual(MIN_SECOND_HALF_BYTES); }); });