From 6398e9a2c6189d10a87dbc710fac222a3fc72070 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 6 Sep 2026 07:11:37 +0000 Subject: [PATCH 1/6] usockets: stagger connect attempts to the addresses of a name (RFC 8305) Dial one address at a time. Start the next address 250 ms after the previous one while the earlier attempts are still pending, or at once when an attempt fails. The first address to connect wins and the rest are reset. Before, the first four addresses were opened at once and a fifth was only tried once at most one attempt was left pending. Two black holes plus two refused addresses in the first four parked the dial on the kernel's SYN retries for about two minutes. A name with three addresses also cost the server three accepted connections, two of them reset. --- packages/bun-usockets/src/context.c | 36 +++-- .../bun-usockets/src/eventing/epoll_kqueue.c | 9 +- packages/bun-usockets/src/internal/internal.h | 19 ++- .../bun-usockets/src/internal/loop_data.h | 7 + packages/bun-usockets/src/loop.c | 87 +++++++++++ src/runtime/dns_jsc/dns.rs | 5 +- src/uws_sys/InternalLoopData.rs | 5 + test/js/bun/dns/dns-interleave.test.ts | 145 ++++++++++++------ 8 files changed, 250 insertions(+), 63 deletions(-) diff --git a/packages/bun-usockets/src/context.c b/packages/bun-usockets/src/context.c index 2035ce089304..11cb2c408880 100644 --- a/packages/bun-usockets/src/context.c +++ b/packages/bun-usockets/src/context.c @@ -24,7 +24,6 @@ #include #include #endif -#define CONCURRENT_CONNECTIONS 4 #if defined(BUN_DEBUG) || (defined(__has_feature) && __has_feature(address_sanitizer)) || defined(__SANITIZE_ADDRESS__) #include @@ -692,11 +691,17 @@ struct us_socket_t *us_socket_group_connect_unix(struct us_socket_group_t *group return connect_socket; } -int start_connections(struct us_connecting_socket_t *c, int count) { +/* One address at a time, the way RFC 8305 §5 dials: the next address starts + * LIBUS_CONNECT_ATTEMPT_DELAY_NS after the previous one, or at once when an + * attempt fails. The first to connect wins and the rest are reset. Opening + * every address at once instead would park the dial on the kernel's SYN + * retries (about two minutes) whenever the first few are black holes, and + * would cost the server one accepted connection per address. */ +int us_internal_socket_start_next_attempt(struct us_connecting_socket_t *c) { int opened = 0; struct us_socket_group_t *group = c->group; struct us_loop_t *loop = group->loop; - for (; c->addrinfo_head != NULL && opened < count; c->addrinfo_head = c->addrinfo_head->ai_next) { + for (; c->addrinfo_head != NULL && opened < 1; c->addrinfo_head = c->addrinfo_head->ai_next) { struct sockaddr_storage addr; init_addr_with_port(c->addrinfo_head, c->port, &addr); /* The deferred-DNS path does not carry a local binding. */ @@ -765,13 +770,14 @@ void us_internal_socket_after_resolve(struct us_connecting_socket_t *c) { c->addrinfo_head = &result->entries->info; - int opened = start_connections(c, CONCURRENT_CONNECTIONS); - if (opened == 0) { + if (!us_internal_socket_start_next_attempt(c)) { /* Same as the exhausted path in us_internal_socket_after_open: a * real connect failure must not be reported as a caller abort. */ c->error = ECONNREFUSED; us_connecting_socket_close(c); + return; } + us_internal_connect_attempt_schedule(c); } void us_internal_socket_after_open(struct us_socket_t *s, int error) { @@ -809,16 +815,16 @@ void us_internal_socket_after_open(struct us_socket_t *s, int error) { } us_socket_close(s, LIBUS_SOCKET_CLOSE_CODE_CONNECTION_RESET, 0); - if (c->connecting_head == NULL || c->connecting_head->connect_next == NULL) { - int opened = start_connections(c, c->connecting_head == NULL ? CONCURRENT_CONNECTIONS : 1); - if (opened == 0 && c->connecting_head == NULL) { - /* Every resolved address failed to connect. Without this, - * us_connecting_socket_close defaults c->error to - * ECANCELED (caller abort) and never invalidates the - * DNS cache entry for the dead host. */ - c->error = ECONNREFUSED; - us_connecting_socket_close(c); - } + /* A failure advances to the next address at once. */ + if (!us_internal_socket_start_next_attempt(c) && c->connecting_head == NULL) { + /* Every resolved address failed to connect. Without this, + * us_connecting_socket_close defaults c->error to + * ECANCELED (caller abort) and never invalidates the + * DNS cache entry for the dead host. */ + c->error = ECONNREFUSED; + us_connecting_socket_close(c); + } else { + us_internal_connect_attempt_schedule(c); } } else { us_internal_socket_connect_failed(s, error); diff --git a/packages/bun-usockets/src/eventing/epoll_kqueue.c b/packages/bun-usockets/src/eventing/epoll_kqueue.c index 6e603ba7bed4..0f996d94b7e5 100644 --- a/packages/bun-usockets/src/eventing/epoll_kqueue.c +++ b/packages/bun-usockets/src/eventing/epoll_kqueue.c @@ -428,9 +428,14 @@ static void us_internal_drain_ready_polls(struct us_loop_t *loop) { } } -/* Bound `timeout` by the socket-timeout sweep deadline (NULL == forever). */ +/* Bound `timeout` by the socket-timeout sweep deadline and the next scheduled + * connect attempt (NULL == forever). */ static const struct timespec *us_internal_clamp_to_sweep(struct us_loop_t *loop, const struct timespec *timeout, struct timespec *storage) { long long ns = us_internal_sweep_timeout_ns(loop); + long long connect_ns = us_internal_connect_attempt_timeout_ns(loop); + if (connect_ns >= 0 && (ns < 0 || connect_ns < ns)) { + ns = connect_ns; + } if (ns < 0) { return timeout; } @@ -465,6 +470,7 @@ void us_loop_run(struct us_loop_t *loop) { us_internal_dispatch_ready_polls(loop); us_internal_drain_ready_polls(loop); us_internal_sweep_if_due(loop); + us_internal_connect_attempts_if_due(loop); /* Emit post callback */ us_internal_loop_post(loop); @@ -558,6 +564,7 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout us_internal_dispatch_ready_polls(loop); us_internal_drain_ready_polls(loop); us_internal_sweep_if_due(loop); + us_internal_connect_attempts_if_due(loop); /* Emit post callback */ us_internal_loop_post(loop); diff --git a/packages/bun-usockets/src/internal/internal.h b/packages/bun-usockets/src/internal/internal.h index a5c8c1536d91..d5be279c9df8 100644 --- a/packages/bun-usockets/src/internal/internal.h +++ b/packages/bun-usockets/src/internal/internal.h @@ -182,13 +182,26 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int eof, in void us_internal_timer_sweep(us_loop_r loop); void us_internal_enable_sweep_timer(struct us_loop_t *loop); void us_internal_disable_sweep_timer(struct us_loop_t *loop); -#ifndef LIBUS_USE_LIBUV /* CLOCK_MONOTONIC in ns. The clock every deadline on the loop is measured * against, so anything comparing against one must read it and not another. */ uint64_t us_internal_monotonic_ns(void); +#ifndef LIBUS_USE_LIBUV long long us_internal_sweep_timeout_ns(struct us_loop_t *loop); void us_internal_sweep_if_due(struct us_loop_t *loop); +/* ns until the earliest scheduled connect attempt on this loop, or -1. */ +long long us_internal_connect_attempt_timeout_ns(struct us_loop_t *loop); #endif +/* Delay between two connect attempts to the addresses of one name while the + * earlier attempts are still pending (RFC 8305 §5 "Connection Attempt Delay"). */ +#define LIBUS_CONNECT_ATTEMPT_DELAY_NS (250LL * 1000000LL) +/* Schedule the next address of `c` for now + LIBUS_CONNECT_ATTEMPT_DELAY_NS. + * A no-op when `c` has no address left. */ +void us_internal_connect_attempt_schedule(struct us_connecting_socket_t *c); +/* Start the scheduled attempt of every connecting socket whose delay elapsed. */ +void us_internal_connect_attempts_if_due(struct us_loop_t *loop); +/* Open a connect() to the next address of `c`, at most one. Returns 1 when a + * socket was opened, 0 when every remaining address failed to open. */ +int us_internal_socket_start_next_attempt(struct us_connecting_socket_t *c); void us_internal_free_closed_sockets(us_loop_r loop); /* Defers closing `s` with `code` to us_internal_loop_post. */ void us_internal_loop_close_unresumable_socket(struct us_loop_t *loop, struct us_socket_t *s, int code); @@ -452,6 +465,10 @@ struct us_connecting_socket_t { uint16_t port; int error; struct addrinfo *addrinfo_head; + /* Absolute monotonic ns at which the next address in addrinfo_head is + * tried while the current attempts are still pending (RFC 8305 §5), or 0 + * when nothing is scheduled. */ + long long next_attempt_ns; // this is used to track pending connecting sockets in the context struct us_connecting_socket_t* next_pending; struct us_connecting_socket_t* prev_pending; diff --git a/packages/bun-usockets/src/internal/loop_data.h b/packages/bun-usockets/src/internal/loop_data.h index 4b6ef00d6f76..ef5759462955 100644 --- a/packages/bun-usockets/src/internal/loop_data.h +++ b/packages/bun-usockets/src/internal/loop_data.h @@ -90,6 +90,13 @@ struct us_internal_loop_data_t { int low_prio_budget; struct us_connecting_socket_t *dns_ready_head; struct us_connecting_socket_t *closed_connecting_head; + /* Absolute monotonic ns of the earliest us_connecting_socket_t::next_attempt_ns + * on this loop, or -1. POSIX folds it into the poll timeout next to the + * sweep deadline; libuv arms connect_timer to it. */ + long long connect_next_attempt_ns; +#ifdef LIBUS_USE_LIBUV + struct us_timer_t *connect_timer; +#endif zig_mutex_t mutex; void *parent_ptr; char parent_tag; diff --git a/packages/bun-usockets/src/loop.c b/packages/bun-usockets/src/loop.c index b80076e5c142..5bd2a2a88f41 100644 --- a/packages/bun-usockets/src/loop.c +++ b/packages/bun-usockets/src/loop.c @@ -55,6 +55,10 @@ void sweep_timer_cb(struct us_internal_callback_t *cb); // when the sweep timer is disabled, we don't need to do anything void sweep_timer_noop(struct us_timer_t *timer) {} +uint64_t us_internal_monotonic_ns(void) { + return uv_hrtime(); +} + void us_internal_enable_sweep_timer(struct us_loop_t *loop) { loop->data.sweep_timer_count++; if (loop->data.sweep_timer_count == 1) { @@ -118,7 +122,87 @@ void us_internal_sweep_if_due(struct us_loop_t *loop) { us_internal_timer_sweep(loop); } +long long us_internal_connect_attempt_timeout_ns(struct us_loop_t *loop) { + if (loop->data.connect_next_attempt_ns < 0) { + return -1; + } + long long diff = loop->data.connect_next_attempt_ns - (long long) us_internal_monotonic_ns(); + return diff > 0 ? diff : 0; +} + +#endif + +/* ── Connection attempt delay (RFC 8305 §5) ─────────────────────────────── */ + +#ifdef LIBUS_USE_LIBUV +static void connect_timer_cb(struct us_internal_callback_t *cb) { + us_internal_connect_attempts_if_due(cb->loop); +} + +/* Arm (or re-arm) the uv timer for loop->data.connect_next_attempt_ns. */ +static void us_internal_connect_timer_arm(struct us_loop_t *loop) { + long long deadline = loop->data.connect_next_attempt_ns; + if (deadline < 0) { + us_timer_set(loop->data.connect_timer, (void (*)(struct us_timer_t *)) connect_timer_cb, 0, 0); + return; + } + long long ms = (deadline - (long long) us_internal_monotonic_ns()) / 1000000LL; + us_timer_set(loop->data.connect_timer, (void (*)(struct us_timer_t *)) connect_timer_cb, ms > 0 ? (int) ms : 1, 0); +} +#endif + +void us_internal_connect_attempt_schedule(struct us_connecting_socket_t *c) { + if (c->addrinfo_head == NULL) { + c->next_attempt_ns = 0; + return; + } + struct us_loop_t *loop = c->loop; + c->next_attempt_ns = (long long) us_internal_monotonic_ns() + LIBUS_CONNECT_ATTEMPT_DELAY_NS; + /* Every delay is the same length, so a new deadline is never earlier than + * one already armed. */ + if (loop->data.connect_next_attempt_ns < 0) { + loop->data.connect_next_attempt_ns = c->next_attempt_ns; +#ifdef LIBUS_USE_LIBUV + us_internal_connect_timer_arm(loop); +#endif + } +} + +void us_internal_connect_attempts_if_due(struct us_loop_t *loop) { + if (loop->data.connect_next_attempt_ns < 0) { + return; + } + long long now = (long long) us_internal_monotonic_ns(); + if (now < loop->data.connect_next_attempt_ns) { + return; + } + long long next = -1; + for (struct us_socket_group_t *group = loop->data.head; group; group = group->next) { + for (struct us_connecting_socket_t *c = group->head_connecting_sockets; c; c = c->next_pending) { + if (c->next_attempt_ns <= 0) { + continue; + } + /* A connecting socket with a scheduled attempt always has at least + * one attempt pending: the failure path of the last pending + * attempt starts the next address itself and decides the outcome. + * So nothing here can close `c` or dispatch to the owner, and the + * lists stay intact while they are walked. */ + if (c->next_attempt_ns <= now && c->connecting_head != NULL) { + c->next_attempt_ns = 0; + if (us_internal_socket_start_next_attempt(c)) { + us_internal_connect_attempt_schedule(c); + } + } + if (c->next_attempt_ns > 0 && (next < 0 || c->next_attempt_ns < next)) { + next = c->next_attempt_ns; + } + } + } + loop->data.connect_next_attempt_ns = next; +#ifdef LIBUS_USE_LIBUV + us_internal_connect_timer_arm(loop); #endif +} /* -1 if the wakeup async cannot be created; nothing is left allocated in loop->data. */ @@ -127,9 +211,11 @@ int us_internal_loop_data_init(struct us_loop_t *loop, void (*wakeup_cb)(struct // We allocate with calloc, so we only need to initialize the specific fields in use. #ifdef LIBUS_USE_LIBUV loop->data.sweep_timer = us_create_timer(loop, 1, 0); + loop->data.connect_timer = us_create_timer(loop, 1, 0); #else loop->data.sweep_next_tick_ns = -1; #endif + loop->data.connect_next_attempt_ns = -1; loop->data.sweep_timer_count = 0; loop->data.recv_buf = us_malloc(LIBUS_RECV_BUFFER_LENGTH + LIBUS_RECV_BUFFER_PADDING * 2); loop->data.send_buf = us_malloc(LIBUS_SEND_BUFFER_LENGTH); @@ -167,6 +253,7 @@ void us_internal_loop_data_free(struct us_loop_t *loop) { #ifdef LIBUS_USE_LIBUV us_timer_close(loop->data.sweep_timer, 0); + us_timer_close(loop->data.connect_timer, 0); if (loop->data.quic_timer) us_timer_close(loop->data.quic_timer, 0); #endif us_internal_async_close(loop->data.wakeup_async); diff --git a/src/runtime/dns_jsc/dns.rs b/src/runtime/dns_jsc/dns.rs index 74e2b72876cd..b12d118e3183 100644 --- a/src/runtime/dns_jsc/dns.rs +++ b/src/runtime/dns_jsc/dns.rs @@ -2516,8 +2516,9 @@ pub(crate) mod internal { } // Pack getaddrinfo results into one allocation with address families - // interleaved (RFC 8305 §4) so an unroutable family can never fill all - // CONCURRENT_CONNECTIONS parallel connect attempts. See #4938 / #33278. + // interleaved (RFC 8305 §4) so the staggered connect attempts in + // usockets reach the other family on the second attempt when the first + // family is unroutable. See #4938 / #33278. fn process_results(info: *mut AddrInfo) -> Box<[ResultEntry]> { let mut count: usize = 0; let mut n_first: usize = 0; diff --git a/src/uws_sys/InternalLoopData.rs b/src/uws_sys/InternalLoopData.rs index e1777af620e2..800976cdee6c 100644 --- a/src/uws_sys/InternalLoopData.rs +++ b/src/uws_sys/InternalLoopData.rs @@ -50,6 +50,11 @@ pub struct InternalLoopData { pub low_prio_budget: i32, pub dns_ready_head: *mut ConnectingSocket, pub closed_connecting_head: *mut ConnectingSocket, + /// Earliest scheduled connect attempt (monotonic ns) on this loop, or -1. + /// Owned by `loop.c`. Never read here. + pub(crate) connect_next_attempt_ns: i64, + #[cfg(windows)] + pub(crate) connect_timer: *mut Timer, /// `bun.Mutex.ReleaseImpl.Type` — must match the C-side `zig_mutex_t` /// (`packages/bun-usockets/src/internal/loop_data.h`). `Bun__lock`/`Bun__unlock` /// are called on this field by C, and `loop.c` runtime-checks diff --git a/test/js/bun/dns/dns-interleave.test.ts b/test/js/bun/dns/dns-interleave.test.ts index da288b466c56..fe2e0d1d22cf 100644 --- a/test/js/bun/dns/dns-interleave.test.ts +++ b/test/js/bun/dns/dns-interleave.test.ts @@ -53,6 +53,49 @@ async function run(body: string, timeoutMs = 10_000) { return { out, stderr, exitCode, signal: proc.signalCode }; } +// The backlog-0 trick and 127.0.0.0/8-on-lo are Linux-specific; the raw setup +// dlopens glibc. listen(fd, 0) + a full one-slot accept queue makes the kernel +// drop every further SYN to ip:port, the same EINPROGRESS-forever a filtered +// network produces. +const blackholePrelude = /* js */ ` +const { dlopen } = require("bun:ffi"); +const libc = dlopen("libc.so.6", { + socket: { args: ["int","int","int"], returns: "int" }, + bind: { args: ["int","ptr","int"], returns: "int" }, + listen: { args: ["int","int"], returns: "int" }, + connect: { args: ["int","ptr","int"], returns: "int" }, + close: { args: ["int"], returns: "int" }, + setsockopt:{args: ["int","int","int","ptr","int"],returns:"int" }, +}); +const AF_INET=2, SOCK_STREAM=1, SOCK_NONBLOCK=0o4000, SOL_SOCKET=1, SO_REUSEADDR=2; +function sockaddr_in(ip, port) { + const b = new Uint8Array(16); + new DataView(b.buffer).setUint16(0, AF_INET, true); + b[2] = (port>>8)&0xff; b[3] = port&0xff; + const o = ip.split(".").map(Number); b[4]=o[0]; b[5]=o[1]; b[6]=o[2]; b[7]=o[3]; + return b; +} +const fds = []; +// listen(fd, 0) + fill the one-slot accept queue so further SYNs to +// ip:port are silently dropped (same EINPROGRESS a filtered network +// produces). +function blackhole(ip, port) { + const fd = libc.symbols.socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) throw new Error("socket() failed"); + fds.push(fd); + const one = new Int32Array([1]); + libc.symbols.setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, one, 4); + if (libc.symbols.bind(fd, sockaddr_in(ip, port), 16) !== 0) + throw new Error("bind(" + ip + ":" + port + ") failed"); + if (libc.symbols.listen(fd, 0) !== 0) throw new Error("listen() failed"); + for (let i = 0; i < 8; i++) { + const c = libc.symbols.socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); + fds.push(c); + libc.symbols.connect(c, sockaddr_in(ip, port), 16); + } +} +`; + describe.concurrent("getaddrinfo interleave (RFC 8305)", () => { test("fetch() through a seeded 4xAAAA + 4xA entry connects via the interleaved order", async () => { const { out, stderr, exitCode, signal } = await run(/* js */ ` @@ -114,52 +157,13 @@ describe.concurrent("getaddrinfo interleave (RFC 8305)", () => { // End-to-end proof: with the first family blackholed (SYNs silently dropped, // connect() stuck in EINPROGRESS until kernel SYN-retry exhaustion), fetch() - // must still succeed because the interleave put the other family in the - // first CONCURRENT_CONNECTIONS batch. Without the fix all four initial - // attempts are blackholed and the fetch hangs. The backlog-0 trick and - // 127.0.0.0/8-on-lo are Linux-specific; the raw setup dlopens glibc. + // must still succeed because the interleave put the other family second. test.skipIf(!isLinux || isMusl)( "fetch() succeeds when the first family is blackholed and only the other family is reachable", async () => { const { out, stderr, exitCode, signal } = await run( /* js */ ` - const { dlopen } = require("bun:ffi"); - const libc = dlopen("libc.so.6", { - socket: { args: ["int","int","int"], returns: "int" }, - bind: { args: ["int","ptr","int"], returns: "int" }, - listen: { args: ["int","int"], returns: "int" }, - connect: { args: ["int","ptr","int"], returns: "int" }, - close: { args: ["int"], returns: "int" }, - setsockopt:{args: ["int","int","int","ptr","int"],returns:"int" }, - }); - const AF_INET=2, SOCK_STREAM=1, SOCK_NONBLOCK=0o4000, SOL_SOCKET=1, SO_REUSEADDR=2; - function sockaddr_in(ip, port) { - const b = new Uint8Array(16); - new DataView(b.buffer).setUint16(0, AF_INET, true); - b[2] = (port>>8)&0xff; b[3] = port&0xff; - const o = ip.split(".").map(Number); b[4]=o[0]; b[5]=o[1]; b[6]=o[2]; b[7]=o[3]; - return b; - } - const fds = []; - // listen(fd, 0) + fill the one-slot accept queue so further SYNs to - // ip:port are silently dropped (same EINPROGRESS a filtered network - // produces). - function blackhole(ip, port) { - const fd = libc.symbols.socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) throw new Error("socket() failed"); - fds.push(fd); - const one = new Int32Array([1]); - libc.symbols.setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, one, 4); - if (libc.symbols.bind(fd, sockaddr_in(ip, port), 16) !== 0) - throw new Error("bind(" + ip + ":" + port + ") failed"); - if (libc.symbols.listen(fd, 0) !== 0) throw new Error("listen() failed"); - for (let i = 0; i < 8; i++) { - const c = libc.symbols.socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); - fds.push(c); - libc.symbols.connect(c, sockaddr_in(ip, port), 16); - } - } - + ${blackholePrelude} using server = Bun.serve({ port: 0, hostname: "::1", fetch: () => new Response("ok via ::1") }); const port = server.port; const dead = ["127.0.0.2","127.0.0.3","127.0.0.4","127.0.0.5"]; @@ -185,10 +189,10 @@ describe.concurrent("getaddrinfo interleave (RFC 8305)", () => { `, 20_000, ); - // With the broken interleave the order is [4,4,4,4,6]: all four initial + // With the broken interleave the order is [4,4,4,4,6]: the first // attempts sit in EINPROGRESS and the fetch aborts at 4s with - // TimeoutError. With the fix the order is [4,6,4,4,4]: ::1 is attempted - // in the first batch and connects immediately. + // TimeoutError. With the fix the order is [4,6,4,4,4]: ::1 is the + // second attempt and connects. expect({ out, stderr, exitCode, signal }).toEqual({ out: { ok: true, ms: expect.any(Number), order: [4, 6, 4, 4, 4], body: "ok via ::1" }, stderr: expect.any(String), @@ -201,6 +205,59 @@ describe.concurrent("getaddrinfo interleave (RFC 8305)", () => { ); }); +// usockets dials the addresses of a name the way RFC 8305 §5 does: one +// attempt, the next address 250 ms later while the earlier ones are still +// pending, or at once when one fails. Before, it opened the first four at once +// and only went on once at most one of them was left pending, so two black +// holes plus two refused addresses in the first four parked the dial on the +// kernel's SYN retries (about two minutes) with a reachable fifth address +// never tried. +describe.concurrent("connection attempt delay (RFC 8305 §5)", () => { + test.skipIf(!isLinux || isMusl)( + "fetch() reaches the fifth address when two of the first four hang and two are refused", + async () => { + const { out, stderr, exitCode, signal } = await run( + /* js */ ` + ${blackholePrelude} + using server = Bun.serve({ port: 0, hostname: "127.0.0.6", fetch: () => new Response("ok via 127.0.0.6") }); + const port = server.port; + blackhole("127.0.0.2", port); + blackhole("127.0.0.3", port); + // Nothing listens on 127.0.0.4 / 127.0.0.5: connect() fails at once. + const host = "he-stagger-" + port + ".test"; + const order = dnsCacheSeed(host, ["127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5", "127.0.0.6"]); + + const t0 = performance.now(); + let result; + try { + const res = await fetch("http://" + host + ":" + port + "/", { + signal: AbortSignal.timeout(4000), + }); + result = { ok: true, ms: Math.round(performance.now() - t0), order, body: await res.text() }; + } catch (e) { + result = { ok: false, ms: Math.round(performance.now() - t0), order, err: e?.name ?? String(e) }; + } + console.log(JSON.stringify(result)); + for (const fd of fds) libc.symbols.close(fd); + process.exit(0); + `, + 20_000, + ); + // All five are IPv4, so the interleave leaves the order alone. The two + // refused attempts fail at once and advance to the next address each + // time, so 127.0.0.6 is dialed about 500 ms after the first attempt. + expect({ out, stderr, exitCode, signal }).toEqual({ + out: { ok: true, ms: expect.any(Number), order: [4, 4, 4, 4, 4], body: "ok via 127.0.0.6" }, + stderr: expect.any(String), + exitCode: 0, + signal: null, + }); + expect(out.ms).toBeLessThan(4000); + }, + 20_000, + ); +}); + // Loopback: the connect-path resolver asks getaddrinfo with AI_ADDRCONFIG, and // glibc does not count loopback addresses as "configured". On a host whose only // IPv6 address is ::1 (a container on an IPv4-only Docker network) it answers From de56f4c1e34de1a13b69ee9405b611ba820ae9e4 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 6 Sep 2026 07:34:39 +0000 Subject: [PATCH 2/6] usockets(win): re-arm the connect attempt timer when uv fires it early uv_timer_start counts from the cached loop time in whole milliseconds, so the one-shot timer can fire just before the nanosecond deadline. Round the delay up and re-arm from the callback on an early fire. --- packages/bun-usockets/src/loop.c | 7 ++++++- test/js/bun/dns/dns-interleave.test.ts | 14 +++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/bun-usockets/src/loop.c b/packages/bun-usockets/src/loop.c index 5bd2a2a88f41..f5ec800e234b 100644 --- a/packages/bun-usockets/src/loop.c +++ b/packages/bun-usockets/src/loop.c @@ -146,7 +146,9 @@ static void us_internal_connect_timer_arm(struct us_loop_t *loop) { us_timer_set(loop->data.connect_timer, (void (*)(struct us_timer_t *)) connect_timer_cb, 0, 0); return; } - long long ms = (deadline - (long long) us_internal_monotonic_ns()) / 1000000LL; + /* Round up: uv measures from its cached loop time, so a truncated delay + * fires before the deadline. An early fire re-arms from the callback. */ + long long ms = (deadline - (long long) us_internal_monotonic_ns() + 999999LL) / 1000000LL; us_timer_set(loop->data.connect_timer, (void (*)(struct us_timer_t *)) connect_timer_cb, ms > 0 ? (int) ms : 1, 0); } #endif @@ -174,6 +176,9 @@ void us_internal_connect_attempts_if_due(struct us_loop_t *loop) { } long long now = (long long) us_internal_monotonic_ns(); if (now < loop->data.connect_next_attempt_ns) { +#ifdef LIBUS_USE_LIBUV + us_internal_connect_timer_arm(loop); +#endif return; } long long next = -1; diff --git a/test/js/bun/dns/dns-interleave.test.ts b/test/js/bun/dns/dns-interleave.test.ts index fe2e0d1d22cf..392b38269e26 100644 --- a/test/js/bun/dns/dns-interleave.test.ts +++ b/test/js/bun/dns/dns-interleave.test.ts @@ -1,12 +1,12 @@ // The internal DNS cache (used by the usockets connect path for fetch(), -// WebSocket, Bun.connect() and `bun install`) decides which addresses the -// parallel connection attempts usockets opens get to try. +// WebSocket, Bun.connect() and `bun install`) decides the order in which +// usockets dials the addresses of a name, one at a time (RFC 8305 §5). // -// Interleave: it interleaves address families (RFC 8305 §4) so the four -// parallel attempts always cover both families. registry.npmjs.org resolves to +// Interleave: it interleaves address families (RFC 8305 §4) so the second +// attempt is always the other family. registry.npmjs.org resolves to // 12 AAAA + 12 A; on a dual-stack host with blackholed IPv6 a broken interleave -// leaves all four initial attempts on dead IPv6 and every manifest fetch stalls -// for ~100s waiting on kernel SYN-retry exhaustion. +// puts every IPv4 address behind 12 dead IPv6 ones and every manifest fetch +// waits on all of them. // // https://github.com/oven-sh/bun/issues/4938 // https://github.com/oven-sh/bun/issues/33278 @@ -18,7 +18,7 @@ import { bunEnv, bunExe, isLinux, isMusl } from "harness"; // The DNS cache is process-global and a failed connect evicts its entry, so // each scenario runs in its own process. Every test also does a real fetch() // through the seeded entry, which is consumed by usockets via -// Bun__addrinfo_getRequestResult and start_connections(). +// Bun__addrinfo_getRequestResult and us_internal_socket_start_next_attempt(). async function run(body: string, timeoutMs = 10_000) { const fixture = /* js */ ` const { dnsCacheSeed } = require("bun:internal-for-testing"); From 271e7adb48fc3fe510c8fe332468d615a84b398e Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 6 Sep 2026 08:58:25 +0000 Subject: [PATCH 3/6] Tighten two comments --- src/runtime/dns_jsc/dns.rs | 5 ++--- src/uws_sys/InternalLoopData.rs | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/runtime/dns_jsc/dns.rs b/src/runtime/dns_jsc/dns.rs index b12d118e3183..1c08b24d06b4 100644 --- a/src/runtime/dns_jsc/dns.rs +++ b/src/runtime/dns_jsc/dns.rs @@ -2516,9 +2516,8 @@ pub(crate) mod internal { } // Pack getaddrinfo results into one allocation with address families - // interleaved (RFC 8305 §4) so the staggered connect attempts in - // usockets reach the other family on the second attempt when the first - // family is unroutable. See #4938 / #33278. + // interleaved (RFC 8305 §4): usockets dials them one at a time, so the + // second attempt is the other family. See #4938 / #33278. fn process_results(info: *mut AddrInfo) -> Box<[ResultEntry]> { let mut count: usize = 0; let mut n_first: usize = 0; diff --git a/src/uws_sys/InternalLoopData.rs b/src/uws_sys/InternalLoopData.rs index 800976cdee6c..f84932282146 100644 --- a/src/uws_sys/InternalLoopData.rs +++ b/src/uws_sys/InternalLoopData.rs @@ -50,8 +50,7 @@ pub struct InternalLoopData { pub low_prio_budget: i32, pub dns_ready_head: *mut ConnectingSocket, pub closed_connecting_head: *mut ConnectingSocket, - /// Earliest scheduled connect attempt (monotonic ns) on this loop, or -1. - /// Owned by `loop.c`. Never read here. + /// Earliest scheduled connect attempt (monotonic ns), or -1. Owned by `loop.c`. pub(crate) connect_next_attempt_ns: i64, #[cfg(windows)] pub(crate) connect_timer: *mut Timer, From 23f5e16e65e56b27fad928dce521c629719de1b2 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 18 Oct 2026 02:39:44 +0000 Subject: [PATCH 4/6] usockets: test the connection count per dial, close the connect timer when loop init fails A name with three addresses must cost the server one accepted connection per dial. The test counts accepts on a listener for fetch() and Bun.connect(). It runs on every platform. us_internal_loop_data_init can now fail after the timers exist (libuv). Close the connect timer on that path, next to the sweep timer. --- packages/bun-usockets/src/loop.c | 1 + test/js/bun/dns/dns-interleave.test.ts | 61 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/packages/bun-usockets/src/loop.c b/packages/bun-usockets/src/loop.c index f5ec800e234b..81a5780a8a13 100644 --- a/packages/bun-usockets/src/loop.c +++ b/packages/bun-usockets/src/loop.c @@ -235,6 +235,7 @@ int us_internal_loop_data_init(struct us_loop_t *loop, void (*wakeup_cb)(struct us_free(loop->data.send_buf); #ifdef LIBUS_USE_LIBUV us_timer_close(loop->data.sweep_timer, 0); + us_timer_close(loop->data.connect_timer, 0); #endif return -1; } diff --git a/test/js/bun/dns/dns-interleave.test.ts b/test/js/bun/dns/dns-interleave.test.ts index 392b38269e26..28cc57ef8260 100644 --- a/test/js/bun/dns/dns-interleave.test.ts +++ b/test/js/bun/dns/dns-interleave.test.ts @@ -256,6 +256,67 @@ describe.concurrent("connection attempt delay (RFC 8305 §5)", () => { }, 20_000, ); + + // A server pays for every address the client dials. The first address of a + // healthy name connects inside the attempt delay, so the client never opens + // (and then resets) a connection to the others. + test("a name with three addresses costs the server one connection per dial", async () => { + const { out, stderr, exitCode, signal } = await run(/* js */ ` + let accepted = 0; + using server = Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { + open() { accepted++; }, + data(socket) { socket.end("HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\nConnection: close\\r\\n\\r\\nok"); }, + error() {}, + }, + }); + const port = server.port; + const host = "he-census-" + port + ".test"; + dnsCacheSeed(host, ["127.0.0.1", "127.0.0.1", "127.0.0.1"]); + // The listener accepts in arrival order. Once it answers this dial to + // the literal address, it has accepted every connection opened before. + async function acceptedSinceLastCall() { + await (await fetch("http://127.0.0.1:" + port + "/")).text(); + const count = accepted - 1; + accepted = 0; + return count; + } + + const dials = 10; + for (let i = 0; i < dials; i++) { + await (await fetch("http://" + host + ":" + port + "/")).text(); + } + const viaFetch = await acceptedSinceLastCall(); + + for (let i = 0; i < dials; i++) { + const { promise, resolve, reject } = Promise.withResolvers(); + await Bun.connect({ + hostname: host, + port, + socket: { + open(socket) { socket.write("GET / HTTP/1.1\\r\\nHost: census\\r\\n\\r\\n"); }, + data() {}, + close() { resolve(); }, + error(socket, err) { reject(err); }, + connectError(socket, err) { reject(err); }, + }, + }); + await promise; + } + const viaConnect = await acceptedSinceLastCall(); + + console.log(JSON.stringify({ ok: true, dials, viaFetch, viaConnect })); + process.exit(0); + `); + expect({ out, stderr, exitCode, signal }).toEqual({ + out: { ok: true, dials: 10, viaFetch: 10, viaConnect: 10 }, + stderr: "", + exitCode: 0, + signal: null, + }); + }); }); // Loopback: the connect-path resolver asks getaddrinfo with AI_ADDRCONFIG, and From 27645b25b4b62c5fe1dbb540087a62fb856c8c65 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:14:47 +0000 Subject: [PATCH 5/6] usockets: start no connect attempt for a connecting socket that is closing us_connecting_socket_close keeps the socket linked in its group until the owner has heard the error, and the owner can run the loop from that callback. A due attempt delay then opened a new attempt whose connect_state pointed at the socket that is freed right after. Clear the scheduled attempt on close, and skip a closed socket in the walk. --- packages/bun-usockets/src/loop.c | 2 +- packages/bun-usockets/src/socket.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/bun-usockets/src/loop.c b/packages/bun-usockets/src/loop.c index 81a5780a8a13..de44d84d134f 100644 --- a/packages/bun-usockets/src/loop.c +++ b/packages/bun-usockets/src/loop.c @@ -184,7 +184,7 @@ void us_internal_connect_attempts_if_due(struct us_loop_t *loop) { long long next = -1; for (struct us_socket_group_t *group = loop->data.head; group; group = group->next) { for (struct us_connecting_socket_t *c = group->head_connecting_sockets; c; c = c->next_pending) { - if (c->next_attempt_ns <= 0) { + if (c->next_attempt_ns <= 0 || c->closed) { continue; } /* A connecting socket with a scheduled attempt always has at least diff --git a/packages/bun-usockets/src/socket.c b/packages/bun-usockets/src/socket.c index 3ba69a3a2a9c..7d67d54c549e 100644 --- a/packages/bun-usockets/src/socket.c +++ b/packages/bun-usockets/src/socket.c @@ -210,6 +210,9 @@ void us_connecting_socket_free(struct us_connecting_socket_t *c) { void us_connecting_socket_close(struct us_connecting_socket_t *c) { if (c->closed) return; c->closed = 1; + /* `c` stays linked until its owner has heard the error below, and the owner + * may run the loop from that callback. No address is tried from here on. */ + c->next_attempt_ns = 0; for (struct us_socket_t *s = c->connecting_head; s; s = s->connect_next) { us_internal_socket_group_unlink_socket(s->group, s); From 445215e2f9d8d84a40329da1be84650c0d97fc3a Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:51:47 +0000 Subject: [PATCH 6/6] test: count connections per dial, with a bound that a stalled machine cannot break The census test now records the count of each dial and asserts fewer than two connections per dial over ten dials. A dial whose first connect is reported later than the attempt delay opens a second connection, so an exact count could fail on a loaded machine. Dialing every address at once gives three per dial. --- test/js/bun/dns/dns-interleave.test.ts | 33 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/js/bun/dns/dns-interleave.test.ts b/test/js/bun/dns/dns-interleave.test.ts index 28cc57ef8260..e416818042f2 100644 --- a/test/js/bun/dns/dns-interleave.test.ts +++ b/test/js/bun/dns/dns-interleave.test.ts @@ -258,8 +258,8 @@ describe.concurrent("connection attempt delay (RFC 8305 §5)", () => { ); // A server pays for every address the client dials. The first address of a - // healthy name connects inside the attempt delay, so the client never opens - // (and then resets) a connection to the others. + // healthy name connects inside the attempt delay, so the client does not open + // (and then reset) a connection to the others. test("a name with three addresses costs the server one connection per dial", async () => { const { out, stderr, exitCode, signal } = await run(/* js */ ` let accepted = 0; @@ -284,13 +284,14 @@ describe.concurrent("connection attempt delay (RFC 8305 §5)", () => { return count; } - const dials = 10; - for (let i = 0; i < dials; i++) { + const viaFetch = []; + for (let i = 0; i < 10; i++) { await (await fetch("http://" + host + ":" + port + "/")).text(); + viaFetch.push(await acceptedSinceLastCall()); } - const viaFetch = await acceptedSinceLastCall(); - for (let i = 0; i < dials; i++) { + const viaConnect = []; + for (let i = 0; i < 10; i++) { const { promise, resolve, reject } = Promise.withResolvers(); await Bun.connect({ hostname: host, @@ -304,14 +305,26 @@ describe.concurrent("connection attempt delay (RFC 8305 §5)", () => { }, }); await promise; + viaConnect.push(await acceptedSinceLastCall()); } - const viaConnect = await acceptedSinceLastCall(); - console.log(JSON.stringify({ ok: true, dials, viaFetch, viaConnect })); + console.log(JSON.stringify({ ok: true, viaFetch, viaConnect })); process.exit(0); `); - expect({ out, stderr, exitCode, signal }).toEqual({ - out: { ok: true, dials: 10, viaFetch: 10, viaConnect: 10 }, + // Every address at once is 3 connections per dial. One address at a time + // is 1, unless a stalled machine reports the first connect later than the + // attempt delay and the dial opens a second one. So the bound is on the + // sum of the 10 dials: fewer than 2 per dial. + const sum = (counts: number[] = []) => counts.reduce((a, b) => a + b, 0); + expect({ + out, + fewerThanTwoPerDial: { viaFetch: sum(out.viaFetch) < 20, viaConnect: sum(out.viaConnect) < 20 }, + stderr, + exitCode, + signal, + }).toEqual({ + out: { ok: true, viaFetch: expect.any(Array), viaConnect: expect.any(Array) }, + fewerThanTwoPerDial: { viaFetch: true, viaConnect: true }, stderr: "", exitCode: 0, signal: null,