diff --git a/.changeset/stable-endpoint-lifecycle.md b/.changeset/stable-endpoint-lifecycle.md new file mode 100644 index 00000000..30155b8d --- /dev/null +++ b/.changeset/stable-endpoint-lifecycle.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/core': patch +--- + +Keep active channels alive when an endpoint repeatedly disappears from and returns to discovery. Reject `ready()` after driver shutdown while preserving the original cause of an initial discovery failure. diff --git a/.changeset/topic-flush-boundary.md b/.changeset/topic-flush-boundary.md new file mode 100644 index 00000000..0164175c --- /dev/null +++ b/.changeset/topic-flush-boundary.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/topic': patch +--- + +Make `flush()` wait for acknowledgments of writes accepted before the call. Later writes no longer extend a pending flush. Independent flush boundaries survive reconnect and server deduplication; terminal errors still reject unconfirmed flushes. Use `close()` to stop admission and drain the entire writer. diff --git a/.changeset/topic-lifecycle-audit.md b/.changeset/topic-lifecycle-audit.md new file mode 100644 index 00000000..7bb26620 --- /dev/null +++ b/.changeset/topic-lifecycle-audit.md @@ -0,0 +1,11 @@ +--- +'@ydbjs/core': patch +'@ydbjs/fsm': patch +'@ydbjs/topic': patch +--- + +Expire unreassigned reader partitions after reconnect even when their commits arrive after session initialization. Reject non-positive reader buffer limits instead of leaving reads stalled. + +Keep discovered and explicitly pinned connections separate, close only the affected connection during retirement or invalidation, and drain both during driver shutdown. Preserve the configured TLS server name when discovery provides no override, and omit driver options from debug logging. + +Preserve queued items when a paused read is cancelled immediately after resuming. diff --git a/.changeset/topic-linear-batching.md b/.changeset/topic-linear-batching.md new file mode 100644 index 00000000..8562f9a7 --- /dev/null +++ b/.changeset/topic-linear-batching.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/topic': patch +--- + +Avoid repeatedly scanning a partial writer batch on every `write()`. Track the compressed payload size of the unsent queue so readiness checks remain constant-time while preserving timed flushes, byte/count limits and reconnect recovery. diff --git a/.changeset/topic-memory-ownership.md b/.changeset/topic-memory-ownership.md new file mode 100644 index 00000000..7aec8175 --- /dev/null +++ b/.changeset/topic-memory-ownership.md @@ -0,0 +1,11 @@ +--- +'@ydbjs/core': patch +'@ydbjs/fsm': patch +'@ydbjs/topic': patch +--- + +Release cancelled readiness waiters and completed queue items instead of retaining their payloads until the driver or queue closes. + +Keep topic reader buffers encoded until `read()` selects messages for delivery. Preserve the unread tail after a clean close and discard it on destruction or terminal failure, including when a read iterator is suspended. Release stopped partition state without pending commits and callbacks or custom codecs after client shutdown. Preserve delivered transaction offsets until the transaction finishes. + +Read credit still uses each complete server response's `bytesSize`; no credit is inferred from individual message payloads. `maxBufferBytes` does not bound decompressed data or process RSS. Use `read({ limit })` to bound how many messages are decoded in one batch. diff --git a/.changeset/topic-reader-receive-limit.md b/.changeset/topic-reader-receive-limit.md new file mode 100644 index 00000000..7573a709 --- /dev/null +++ b/.changeset/topic-reader-receive-limit.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/topic': patch +--- + +Report oversized gRPC read responses as terminal errors instead of repeatedly reconnecting to the same unread message. Temporary resource exhaustion remains retryable. diff --git a/.changeset/topic-reader-token-renewal.md b/.changeset/topic-reader-token-renewal.md new file mode 100644 index 00000000..2e39365d --- /dev/null +++ b/.changeset/topic-reader-token-renewal.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/topic': patch +--- + +Keep refreshing reader credentials when the server does not acknowledge an unchanged token. diff --git a/.changeset/topic-stream-recovery-contracts.md b/.changeset/topic-stream-recovery-contracts.md new file mode 100644 index 00000000..f6c8cfb3 --- /dev/null +++ b/.changeset/topic-stream-recovery-contracts.md @@ -0,0 +1,5 @@ +--- +'@ydbjs/topic': patch +--- + +Resume pending reads, commits, and writes after transient stream deadlines and send buffered partial batches after reconnect without restarting their batching delay. Preserve reader source settings after caller mutation and contain rejected async acknowledgment observers. Clarify RAW buffer ownership. diff --git a/.changeset/topic-stream-reliability.md b/.changeset/topic-stream-reliability.md new file mode 100644 index 00000000..a6ccb471 --- /dev/null +++ b/.changeset/topic-stream-reliability.md @@ -0,0 +1,17 @@ +--- +'@ydbjs/topic': patch +--- + +Preserve topic delivery guarantees and resource budgets across concurrent calls and reconnects. + +- Prevent an earlier `flush()` completion from resolving a later flush before its messages are acknowledged. +- Release payload references recovered through reconnect deduplication, and always close writer streams and timers after an internal failure. +- Snapshot metadata and dates, avoid retaining oversized backing buffers, and reject invalid dates or sequence numbers before accepting a message. +- Preserve the reader's retained-byte budget across reconnects and account for releases during backoff without granting excess credit to a new stream. +- Honor read cancellation and partition revocation between slices of a response while preserving unread messages and releasing each response's credit once. +- Restore the `TopicTxWriter` type export for consumers that import the transactional writer contract. +- Restore timed batching: send full batches immediately and partial batches on `flushIntervalMs`; explicit `flush()` and `close()` drain without waiting for the interval. Preserve an expired flush deadline while the in-flight window is occupied. +- Keep graceful writer shutdown on the normal connection lifecycle, waiting for every reconnect handshake and cancelling obsolete retry timers after initialization. +- Ignore late stream completions and token refreshes from replaced connections, and prevent opening a stream after cancellation during driver readiness. +- Preserve transaction cancellation when updating read offsets, keep registered reader callbacks stable, and retain falsy destruction reasons in terminal errors. +- Generate default producer identities with `randomUUID()` and make manual flush sequence numbers independent of initialization timing. diff --git a/e2e/topic/read-write.test.ts b/e2e/topic/read-write.test.ts index 5ee949ab..17e9da83 100644 --- a/e2e/topic/read-write.test.ts +++ b/e2e/topic/read-write.test.ts @@ -9,7 +9,6 @@ import { TopicServiceDefinition, } from '@ydbjs/api/topic' import { create } from '@bufbuild/protobuf' -import { once } from 'node:events' // #region setup declare module 'vitest' { @@ -92,6 +91,7 @@ test('writes and reads concurrently', { timeout: 60_000 }, async (tc) => { await using writer = createTopicWriter(driver, { topic: testTopicName, producer: testProducerName, + maxBufferBytes: BigInt(TOTAL_TRAFFIC) + 16n * 1024n * 1024n, maxInflightCount: TOTAL_BATCHES * BATCH_SIZE, }) @@ -105,11 +105,11 @@ test('writes and reads concurrently', { timeout: 60_000 }, async (tc) => { let rb = 0 let ctrl = new AbortController() // linkSignals, not AbortSignal.any (composite signals accumulate listeners). - using combined = linkSignals(tc.signal, ctrl.signal, AbortSignal.timeout(25_000)) + using combined = linkSignals(tc.signal, ctrl.signal) let signal = combined.signal // Producer. - void (async () => { + let producerTask = (async () => { while (wb < TOTAL_TRAFFIC) { if (signal.aborted) break @@ -121,20 +121,19 @@ test('writes and reads concurrently', { timeout: 60_000 }, async (tc) => { } let start = performance.now() - await writer.flush() + await writer.flush(tc.signal) console.log(`write took ${performance.now() - start} ms`) })() // Consumer. - void (async () => { + let consumerTask = (async () => { for await (let batch of reader.read({ signal })) { - let promise = reader.commit(batch) + await reader.commit(batch) rb += MESSAGE_SIZE * batch.length // >=, not ==: at-least-once redelivery can overshoot the total, and an // exact-equality trigger would then never fire and hang the test. if (rb >= TOTAL_TRAFFIC) { - await promise ctrl.abort() break } @@ -142,7 +141,7 @@ test('writes and reads concurrently', { timeout: 60_000 }, async (tc) => { })() let start = Date.now() - await once(ctrl.signal, 'abort') + await Promise.all([producerTask, consumerTask]) await writer.close() await reader.close() diff --git a/packages/core/ARCHITECTURE.md b/packages/core/ARCHITECTURE.md index 1feb4d41..a88ab950 100644 --- a/packages/core/ARCHITECTURE.md +++ b/packages/core/ARCHITECTURE.md @@ -26,9 +26,9 @@ registry, and health. On any change to the routable set it rebuilds an **immutab `acquireNode()` — reads the latest snapshot reference (swapped by `#consume`), selects a `RoutingSnapshot` ref (pure), and lazily materializes a channel. The only per-RPC dispatch is a fire-and-forget `penalize()` / `recover()` (enqueue only; -handled off the hot path). Reads never dispatch; writes never happen inline. `#channels` / `#retired` / -`#pinned` are facade-owned I/O the transition never touches — that is what makes the -sync hot path race-free against the async FSM. +handled off the hot path). Reads never dispatch; writes never happen inline. + +The runtime keeps discovered physical channels in one map across retirement and revival. The registry owns their active/retired state; retirement does not move the connection between stores. Explicitly pinned channels have a separate cache, even when a pin and discovery share a nodeId. Discovery retirement and pin invalidation each close only their own channel; pool shutdown drains and closes both. ### States (discovery lifecycle) @@ -47,7 +47,7 @@ Global guard: `endpoints.destroy` from any non-terminal state → `closed` (imme | idle | discovery.start | discovering | run first round | | idle | pin | idle | rebuild snapshot | | idle | close | closed | close-before-start | -| discovering | round_succeeded | ready / degraded | apply round, ready-latch, arm interval+idle_sweep | +| discovering | round_succeeded | ready / degraded | apply round, settle readiness waiters, arm interval+idle_sweep | | discovering | round_succeeded (0 endpoints) | discovering | rejected as retryable failure — arm backoff | | discovering | round_failed (retryable) | discovering | arm backoff, stay | | discovering | round_failed (non-retryable) | closed | emit `failed` (only terminal-failure path) | @@ -129,10 +129,7 @@ hard-pin, and the pile-relaxed last-resort tiers still route if every pile is un A connection dropped from discovery is **not** torn down while it works: live streams drain on it and a brief flap does not close it. New RPCs are simply not routed there. -The `idle_sweep` effect closes a retired channel only on genuine breakage -(`SHUTDOWN` / sustained `TRANSIENT_FAILURE`) or after `retiredGraceMs` idle with no -reappearance; a returning node revives the **same** channel. Still-discovered channels -are never proactively closed — grpc-js manages their idle socket. +The `idle_sweep` effect keeps retired channels in `READY`, removes `SHUTDOWN` channels immediately, and reaps other connectivity states after `retiredGraceMs` from retirement. A returning node reuses the same channel; another retirement starts a new grace interval. Still-discovered channels are never proactively closed — grpc-js manages their idle socket. ### Direct topic IO diff --git a/packages/core/src/conn.ts b/packages/core/src/conn.ts index 6febdff1..009dee0d 100644 --- a/packages/core/src/conn.ts +++ b/packages/core/src/conn.ts @@ -52,7 +52,9 @@ export class GrpcConnection implements Connection { ...channelOptions, // Required when the TLS certificate CN doesn't match the gRPC endpoint // address (common in YDB deployments behind a load balancer). - 'grpc.ssl_target_name_override': endpoint.sslTargetNameOverride, + ...(endpoint.sslTargetNameOverride && { + 'grpc.ssl_target_name_override': endpoint.sslTargetNameOverride, + }), }) } diff --git a/packages/core/src/driver.discovery.test.ts b/packages/core/src/driver.discovery.test.ts new file mode 100644 index 00000000..83f71b44 --- /dev/null +++ b/packages/core/src/driver.discovery.test.ts @@ -0,0 +1,117 @@ +import { channel } from 'node:diagnostics_channel' +import { setTimeout as delay } from 'node:timers/promises' + +import { create } from '@bufbuild/protobuf' +import { anyPack } from '@bufbuild/protobuf/wkt' +import { DiscoveryServiceDefinition, ListEndpointsResultSchema } from '@ydbjs/api/discovery' +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { createServer } from 'nice-grpc' +import { expect, test } from 'vitest' + +import { Driver } from './driver.ts' + +// Controlled discovery can remove a node while its real RPC remains live. +test.for([false, true])( + 'rediscovery preserves the active call and reroutes new calls (GOAWAY: %s)', + { timeout: 10_000 }, + async (goaway, tc) => { + let entered = Promise.withResolvers() + let release = Promise.withResolvers() + await using servers = { + first: createServer(), + second: createServer(), + async [Symbol.asyncDispose]() { + release.resolve() + await Promise.all([this.first.shutdown(), this.second.shutdown()]) + }, + } + let callsOnOriginal = 0 + let endpoints: { nodeId: number; address: string; port: number }[] = [] + servers.first.add( + { + listEndpoints: DiscoveryServiceDefinition.listEndpoints, + whoAmI: DiscoveryServiceDefinition.whoAmI, + }, + { + async listEndpoints() { + return { + operation: { + ready: true, + status: StatusIds_StatusCode.SUCCESS, + result: anyPack( + ListEndpointsResultSchema, + create(ListEndpointsResultSchema, { endpoints }) + ), + }, + } + }, + async whoAmI() { + if (++callsOnOriginal === 1) { + entered.resolve() + await release.promise + } + return { operation: { id: 'original' } } + }, + } + ) + servers.second.add( + { whoAmI: DiscoveryServiceDefinition.whoAmI }, + { + async whoAmI() { + return { operation: { id: 'replacement' } } + }, + } + ) + let firstPort = await servers.first.listen('127.0.0.1:0') + let secondPort = await servers.second.listen('127.0.0.1:0') + endpoints = [{ nodeId: 1, address: '127.0.0.1', port: firstPort }] + using driver = new Driver(`grpc://127.0.0.1:${firstPort}/local`, { + 'ydb.sdk.discovery_interval_ms': 200, + 'ydb.sdk.discovery_timeout_ms': 100, + 'ydb.sdk.connection_idle_interval_ms': 10, + 'ydb.sdk.connection_idle_timeout_ms': 0, + }) + await driver.ready(tc.signal) + let retired = channel('ydb:driver.connection.retired') + let onRetired = (event: unknown) => { + let info = event as { driver: unknown; nodeId: bigint } + if (info.driver === driver.identity && info.nodeId === 1n) { + observed.retired = true + } + } + using observed = { + retired: false, + [Symbol.dispose]() { + retired.unsubscribe(onRetired) + }, + } + retired.subscribe(onRetired) + + let client = driver.createClient(DiscoveryServiceDefinition) + let completed = false + let original = client.whoAmI({}, { signal: tc.signal }) + void original.then( + () => { + completed = true + return true + }, + () => { + completed = true + return true + } + ) + await entered.promise + endpoints = [{ nodeId: 2, address: '127.0.0.1', port: secondPort }] + await expect.poll(() => observed.retired, { timeout: 5000 }).toBe(true) + + let replacement = await client.whoAmI({}, { signal: tc.signal }) + expect(replacement.operation?.id).toBe('replacement') + let shutdown = goaway ? servers.first.shutdown() : undefined + // Let retirement sweeps run after GOAWAY while the existing RPC is still active. + await delay(100, undefined, { signal: tc.signal }) + expect(completed).toBe(false) + release.resolve() + await expect(original).resolves.toMatchObject({ operation: { id: 'original' } }) + await shutdown + } +) diff --git a/packages/core/src/driver.pinning.test.ts b/packages/core/src/driver.pinning.test.ts index 97f6765e..4afd7072 100644 --- a/packages/core/src/driver.pinning.test.ts +++ b/packages/core/src/driver.pinning.test.ts @@ -217,3 +217,34 @@ test('keeps pin confirmations ordered when an earlier client is disposed', async expect(response.operation?.id).toBe('discovery') expect(fixture.calls).toBe(0) }) + +test.for([false, true])( + 'routes pinned and discovered clients independently (pin first: %s)', + async (pinFirst, tc) => { + await using fixture = await pinFixture() + await fixture.driver.ready(tc.signal) + + let discovered = fixture.driver.createClient(DiscoveryServiceDefinition, 1n) + using pinned = fixture.driver.createClient(DiscoveryServiceDefinition, { + ...fixture.target, + nodeId: 1n, + }) + + let clients = pinFirst ? [pinned, discovered] : [discovered, pinned] + let responses = [] + + for (let client of clients) { + // oxlint-disable-next-line no-await-in-loop + let response = await client.whoAmI({}, { signal: tc.signal }) + responses.push(response.operation?.id) + } + + expect(responses).toEqual(pinFirst ? ['direct', 'discovery'] : ['discovery', 'direct']) + + pinned[Symbol.dispose]() + await setImmediate() + + let response = await discovered.whoAmI({}, { signal: tc.signal }) + expect(response.operation?.id).toBe('discovery') + } +) diff --git a/packages/core/src/driver.test.ts b/packages/core/src/driver.test.ts index 813a25e8..d4d2634c 100644 --- a/packages/core/src/driver.test.ts +++ b/packages/core/src/driver.test.ts @@ -372,3 +372,37 @@ test('survives background rediscovery failure and keeps serving requests', async await server.shutdown() } }) + +test('rejects ready after close with discovery disabled', async (tc) => { + using driver = new Driver('grpc://127.0.0.1:1/local', { + 'ydb.sdk.enable_discovery': false, + }) + await driver.ready(tc.signal) + driver.close() + await expect(driver.ready(tc.signal)).rejects.toThrow(/closed/i) +}) + +test('rejects ready after close with discovery enabled', async (tc) => { + await using server = await startBadDiscovery({ + status: StatusIds_StatusCode.SUCCESS, + ready: true, + result: anyPack( + ListEndpointsResultSchema, + create(ListEndpointsResultSchema, { + endpoints: [{ nodeId: 1, address: '127.0.0.1', port: 2136 }], + }) + ), + }) + using driver = new Driver(`grpc://127.0.0.1:${server.port}/local`) + await driver.ready(tc.signal) + driver.close() + await expect(driver.ready(tc.signal)).rejects.toThrow(/closed/i) +}) + +test('honors an already aborted ready signal with discovery disabled', async () => { + using driver = new Driver('grpc://127.0.0.1:1/local', { + 'ydb.sdk.enable_discovery': false, + }) + let reason = new Error('caller cancelled readiness') + await expect(driver.ready(AbortSignal.abort(reason))).rejects.toBe(reason) +}) diff --git a/packages/core/src/driver.tls.test.ts b/packages/core/src/driver.tls.test.ts new file mode 100644 index 00000000..1ac4525a --- /dev/null +++ b/packages/core/src/driver.tls.test.ts @@ -0,0 +1,71 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' + +import { DiscoveryServiceDefinition } from '@ydbjs/api/discovery' +import { ServerCredentials } from '@grpc/grpc-js' +import { createServer } from 'nice-grpc' +import { expect, test } from 'vitest' + +import { Driver } from './driver.ts' + +test('uses the configured TLS identity when the endpoint has no override', async (tc) => { + // A private test CA and a mismatched dial hostname isolate TLS identity from cluster configuration. + await using files = { + path: await mkdtemp(join(tmpdir(), 'ydb-driver-tls-')), + [Symbol.asyncDispose]() { + return rm(this.path, { recursive: true, force: true }) + }, + } + let key = join(files.path, 'key.pem') + let certificate = join(files.path, 'cert.pem') + await promisify(execFile)( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + key, + '-out', + certificate, + '-subj', + '/CN=ydb.test', + '-addext', + 'subjectAltName=DNS:ydb.test', + '-days', + '1', + ], + { signal: tc.signal } + ) + let ca = await readFile(certificate) + await using fixture = { + server: createServer(), + [Symbol.asyncDispose]() { + return this.server.shutdown() + }, + } + let server = fixture.server + let service = { whoAmI: DiscoveryServiceDefinition.whoAmI } + server.add(service, { + async whoAmI() { + return {} + }, + }) + let port = await server.listen( + '127.0.0.1:0', + ServerCredentials.createSsl(null, [{ private_key: await readFile(key), cert_chain: ca }]) + ) + using driver = new Driver(`grpcs://127.0.0.1:${port}/local`, { + 'ydb.sdk.enable_discovery': false, + secureOptions: { ca }, + channelOptions: { 'grpc.ssl_target_name_override': 'ydb.test' }, + }) + await expect( + driver.createClient(service).whoAmI({}, { signal: tc.signal }) + ).resolves.toBeDefined() +}) diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index 1931bb79..ef79e71d 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -212,10 +212,6 @@ export class Driver implements Disposable, AsyncDisposable { #middleware: ClientMiddleware #discoveryClient: Client | undefined - // Ready latch for the discovery-DISABLED path only (resolved immediately at - // construction, rejected on close). The enabled path delegates to the pool. - #ready: PromiseWithResolvers = Promise.withResolvers() - #credentialsProvider: CredentialsProvider = new AnonymousCredentialsProvider() #libraries: Set = new Set() @@ -228,14 +224,8 @@ export class Driver implements Disposable, AsyncDisposable { #identity!: DriverIdentity constructor(connectionString: string, userOptions: Readonly = defaultOptions) { - dbg.log('Driver(connectionString: %s, options: %o)', connectionString, userOptions) - this.#initAt = performance.now() - // close() rejects #ready to unblock awaiters; silence unhandled - // rejection when no one observes the promise. - this.#ready.promise.catch(() => {}) - this.cs = this.#parseConnectionString(connectionString) this.options = this.#mergeOptions(userOptions) this.#assertDiscoveryTimings() @@ -324,11 +314,12 @@ export class Driver implements Disposable, AsyncDisposable { using linkedSignal = linkSignals(signal, AbortSignal.timeout(timeout)) try { + linkedSignal.signal.throwIfAborted() + if (this.#closed) throw new Error('Driver is closed') if (this.#endpoints) { await this.#endpoints.pool.ready(linkedSignal.signal) - } else { - await abortable(linkedSignal.signal, this.#ready.promise) } + if (this.#closed) throw new Error('Driver is closed') dbg.log('driver is ready') } catch (error) { @@ -463,7 +454,6 @@ export class Driver implements Disposable, AsyncDisposable { } #markReadyDisabled(): void { - this.#ready.resolve() this.#readyAt = performance.now() let duration = this.#readyAt - this.#initAt @@ -473,8 +463,6 @@ export class Driver implements Disposable, AsyncDisposable { } #markClosedDisabled(): void { - this.#ready.reject(new Error('driver closed')) - let uptime = this.#readyAt ? performance.now() - this.#readyAt : 0 dc('ydb:driver.closed').publish({ driver: this.identity, uptime }) diff --git a/packages/core/src/endpoints/endpoints-runtime.ts b/packages/core/src/endpoints/endpoints-runtime.ts index 48c70fc0..754c8fb1 100644 --- a/packages/core/src/endpoints/endpoints-runtime.ts +++ b/packages/core/src/endpoints/endpoints-runtime.ts @@ -12,7 +12,7 @@ import { channel as dc, tracingChannel } from 'node:diagnostics_channel' import { create } from '@bufbuild/protobuf' import { connectivityState } from '@grpc/grpc-js' -import { abortable, linkSignals } from '@ydbjs/abortable' +import { linkSignals } from '@ydbjs/abortable' import { PileState_State } from '@ydbjs/api/bridge' import { EndpointInfoSchema } from '@ydbjs/api/discovery' import type { ListEndpointsResult, EndpointInfo as ProtoEndpointInfo } from '@ydbjs/api/discovery' @@ -89,13 +89,9 @@ type EndpointsEnv = { // I/O-owned channel state — NEVER touched by the transition. channels: Map - retiredChannels: Map pinnedChannels: Map banStart: Map retiredAt: Map - // When a retired channel was first seen in TRANSIENT_FAILURE — used to require - // a SUSTAINED failure (not one blip) before reaping it. - transientSince: Map // Per-node in-flight RPC count, maintained by BalancedChannel around each call. // Lets graceful close finalize as soon as a channel's streams have drained. inflight: Map @@ -112,7 +108,7 @@ type EndpointsEnv = { backoffMaxMs: number ac: AbortController - readyDeferred: PromiseWithResolvers + readyWaiters: Set> closedDeferred: PromiseWithResolvers isFinalized: boolean timers: Map> @@ -148,40 +144,25 @@ let clearTimerByKey = function clearTimerByKey(env: EndpointsEnv, key: string): } } -// Physically close a materialized channel. `store: 'pinned'` closes only the pin -// (leaving a discovered channel that shares the nodeId intact); otherwise the -// channel is dropped from whichever store holds it. No diagnostics here — the FSM -// emits the `removed` output and the facade republishes it. +// Discovery and explicit pins may dial different addresses for the same nodeId. +// Close only the selected store during retirement/invalidation, and both at shutdown. let dropChannel = function dropChannel( env: EndpointsEnv, nodeId: bigint, - store: 'any' | 'pinned' = 'any' + store: 'any' | 'discovered' | 'pinned' = 'any' ): void { - if (store === 'pinned') { - let pinned = env.pinnedChannels.get(nodeId) - env.pinnedChannels.delete(nodeId) - if (pinned !== undefined) { - dbg.log('close pinned channel to node %d', nodeId) - pinned.close() - } - return - } - - let conn = - env.channels.get(nodeId) ?? - env.retiredChannels.get(nodeId) ?? - env.pinnedChannels.get(nodeId) - env.channels.delete(nodeId) - env.retiredChannels.delete(nodeId) - env.pinnedChannels.delete(nodeId) - env.banStart.delete(nodeId) - env.retiredAt.delete(nodeId) - env.transientSince.delete(nodeId) - env.inflight.delete(nodeId) - env.draining?.delete(nodeId) - if (conn !== undefined) { - dbg.log('close channel to node %d', nodeId) - conn.close() + let stores = + store === 'any' + ? [env.channels, env.pinnedChannels] + : [store === 'pinned' ? env.pinnedChannels : env.channels] + for (let channels of stores) { + let connection = channels.get(nodeId) + channels.delete(nodeId) + connection?.close() + } + if (store !== 'pinned') { + env.banStart.delete(nodeId) + env.retiredAt.delete(nodeId) } } @@ -194,13 +175,10 @@ let finalizeEnv = function finalizeEnv(env: EndpointsEnv): void { env.draining = undefined for (let handle of env.timers.values()) clearTimeout(handle) env.timers.clear() - for (let nodeId of [ - ...env.channels.keys(), - ...env.retiredChannels.keys(), - ...env.pinnedChannels.keys(), - ]) { + for (let nodeId of [...env.channels.keys(), ...env.pinnedChannels.keys()]) { dropChannel(env, nodeId) } + env.inflight.clear() if (!env.ac.signal.aborted) env.ac.abort(new Error('Endpoints finalized')) } @@ -389,18 +367,14 @@ let effects = { clearTimerByKey(ctx, effect.which) }, - // Retire-to-drain: keep the channel open, move it to the drain-watch set. If - // the node was never dialed there is nothing to drain — tell the FSM it is - // closeable now so the registry entry is reaped instead of leaking forever - // (idle_sweep only scans materialized retired channels). + // The registry owns active/retired state; one cache owns discovered channels + // through every retirement and revival. Undialed endpoints need no drain. 'endpoints.effect.retire_channel': (ctx: FullCtx, effect, runtime) => { let conn = ctx.channels.get(effect.nodeId) if (conn === undefined) { runtime.dispatch({ type: 'endpoints.channel_closeable', nodeId: effect.nodeId }) return } - ctx.channels.delete(effect.nodeId) - ctx.retiredChannels.set(effect.nodeId, conn) ctx.retiredAt.set(effect.nodeId, Date.now()) }, @@ -420,35 +394,24 @@ let effects = { if (busy) ctx.draining!.add(nodeId) else runtime.dispatch({ type: 'endpoints.channel_closeable', nodeId }) } - for (let nodeId of ctx.byNodeId.keys()) { - watch(nodeId, ctx.channels.has(nodeId) || ctx.retiredChannels.has(nodeId)) - } - for (let nodeId of ctx.pinned.keys()) { - watch(nodeId, ctx.pinnedChannels.has(nodeId)) + for (let nodeId of new Set([...ctx.byNodeId.keys(), ...ctx.pinned.keys()])) { + watch(nodeId, ctx.channels.has(nodeId) || ctx.pinnedChannels.has(nodeId)) } }, - // Reap retired channels that are genuinely gone. A working (READY / idle-but- - // reconnectable) channel is KEPT so a returning node reuses it — no churn. - // SHUTDOWN is closed at once; a TRANSIENT_FAILURE must be SUSTAINED past the - // grace window (one blip is absorbed), same as any other non-READY idle state. + // Keep READY channels for bound streams. Other states are reaped after the + // retirement grace; SHUTDOWN is already closed and can be removed immediately. 'endpoints.effect.idle_sweep': (ctx: FullCtx, _effect, runtime) => { let now = Date.now() - for (let [nodeId, conn] of ctx.retiredChannels) { + for (let [nodeId, conn] of ctx.channels) { + if (ctx.byNodeId.get(nodeId)?.subState !== 'retired') continue let state = conn.channel.getConnectivityState(false) - if (state === connectivityState.READY) { - ctx.transientSince.delete(nodeId) - continue - } + if (state === connectivityState.READY) continue if (state === connectivityState.SHUTDOWN) { runtime.dispatch({ type: 'endpoints.channel_closeable', nodeId }) continue } - if (state === connectivityState.TRANSIENT_FAILURE && !ctx.transientSince.has(nodeId)) { - ctx.transientSince.set(nodeId, now) - } - // Non-READY (incl. sustained TRANSIENT_FAILURE) at/past the grace window - // (>= so a zero grace reaps a non-READY channel on the first sweep). + // A zero grace reaps a non-READY channel on the first sweep. if (now - (ctx.retiredAt.get(nodeId) ?? now) >= ctx.retiredGraceMs) { runtime.dispatch({ type: 'endpoints.channel_closeable', nodeId }) } @@ -600,12 +563,31 @@ export class EndpointPool implements Disposable, AsyncDisposable { } async ready(signal?: AbortSignal): Promise { - // readyDeferred is the authoritative latch: #consume resolves it on - // `ready` and rejects it with the real cause on `failed`/`closed`. We do - // NOT link the pool's own ac.signal here — its abort reason ('Endpoints - // finalized') would otherwise race ahead of the true discovery error. - let promise = this.#env.readyDeferred.promise - await (signal !== undefined ? abortable(signal, promise) : promise) + signal?.throwIfAborted() + this.#machine.signal.throwIfAborted() + let state = this.#machine.state + if (state === 'closing') throw new Error('Endpoints closed') + if (this.#env.readyAt === undefined) { + let waiter = Promise.withResolvers() + let onAbort = () => { + this.#env.readyWaiters.delete(waiter) + waiter.reject(signal!.reason) + } + this.#env.readyWaiters.add(waiter) + signal?.addEventListener('abort', onAbort, { once: true }) + try { + await waiter.promise + } finally { + this.#env.readyWaiters.delete(waiter) + signal?.removeEventListener('abort', onAbort) + } + } else { + // A close may start before the caller resumes an already-ready pool. + await Promise.resolve() + } + this.#machine.signal.throwIfAborted() + state = this.#machine.state + if (state === 'closing' || state === 'closed') throw new Error('Endpoints closed') } async close(): Promise { @@ -623,25 +605,13 @@ export class EndpointPool implements Disposable, AsyncDisposable { #materialize(ref: EndpointRef): Connection { let nodeId = ref.nodeId - let existing = - this.#env.channels.get(nodeId) ?? - this.#env.retiredChannels.get(nodeId) ?? - this.#env.pinnedChannels.get(nodeId) + let channels = ref.state === 'pinned' ? this.#env.pinnedChannels : this.#env.channels + let existing = channels.get(nodeId) if (existing !== undefined) return existing let conn = this.#env.connectionFactory(ref) - if (ref.state === 'pinned') { - this.#env.pinnedChannels.set(nodeId, conn) - } else if (ref.state === 'retired') { - // A channel first dialed after the node was retired must land in the - // retired store so idle_sweep governs it. Defensive: a retired ref with - // no channel is reaped before it can be selected. - /* node:coverage ignore start */ - this.#env.retiredChannels.set(nodeId, conn) - this.#env.retiredAt.set(nodeId, Date.now()) - /* node:coverage ignore stop */ - } else { - this.#env.channels.set(nodeId, conn) - } + channels.set(nodeId, conn) + // Affinity can materialize a retired endpoint before its queued reap is applied. + if (ref.state === 'retired') this.#env.retiredAt.set(nodeId, Date.now()) return conn } @@ -659,17 +629,26 @@ export class EndpointPool implements Disposable, AsyncDisposable { /* node:coverage ignore start -- backstop for an unrecoverable FSM fault (should never happen) */ } catch (error) { dbg.log('endpoints machine faulted: %O', error) - env.readyDeferred.reject(error) + this.#rejectReadyWaiters(error) /* node:coverage ignore stop */ } finally { finalizeEnv(env) - this.#rejectPendingPins(this.#machine.signal.reason ?? new Error('Endpoints closed')) - // No-ops if already settled; guarantees no awaiter hangs on a fault. - env.readyDeferred.reject(new Error('Endpoints closed')) + let reason = this.#machine.signal.reason ?? new Error('Endpoints closed') + + this.#rejectReadyWaiters(reason) + this.#rejectPendingPins(reason) env.closedDeferred.resolve() } } + #rejectReadyWaiters(error: unknown): void { + for (let waiter of this.#env.readyWaiters) { + waiter.reject(error) + } + + this.#env.readyWaiters.clear() + } + #rejectPendingPins(reason: unknown): void { for (let pending of this.#pendingPins) { pending.reject(reason) @@ -775,14 +754,15 @@ export class EndpointPool implements Disposable, AsyncDisposable { break case 'endpoints.ready': env.readyAt = Date.now() - env.readyDeferred.resolve() + for (let waiter of env.readyWaiters) waiter.resolve() + env.readyWaiters.clear() dc('ydb:driver.ready').publish({ driver: env.identity, duration: env.readyAt - env.initAt, }) break case 'endpoints.failed': - env.readyDeferred.reject(out.error) + this.#rejectReadyWaiters(out.error) dc('ydb:driver.failed').publish({ driver: env.identity, duration: Date.now() - env.initAt, @@ -790,8 +770,10 @@ export class EndpointPool implements Disposable, AsyncDisposable { }) break case 'endpoints.closed': - this.#rejectPendingPins(out.reason) - env.readyDeferred.reject(new Error('Endpoints closed')) + let reason = this.#machine.signal.reason ?? new Error('Endpoints closed') + + this.#rejectReadyWaiters(reason) + this.#rejectPendingPins(reason) dc('ydb:driver.closed').publish({ driver: env.identity, uptime: env.readyAt !== undefined ? Date.now() - env.readyAt : 0, @@ -895,11 +877,9 @@ export let createEndpointsRuntime = function createEndpointsRuntime( listEndpoints: config.listEndpoints, connectionFactory: config.connectionFactory ?? defaultFactory, channels: new Map(), - retiredChannels: new Map(), pinnedChannels: new Map(), banStart: new Map(), retiredAt: new Map(), - transientSince: new Map(), inflight: new Map(), draining: undefined, discoveryTimeoutMs: config.discoveryTimeoutMs ?? DEFAULT_DISCOVERY_TIMEOUT_MS, @@ -910,7 +890,7 @@ export let createEndpointsRuntime = function createEndpointsRuntime( backoffBaseMs: DEFAULT_BACKOFF_BASE_MS, backoffMaxMs: DEFAULT_BACKOFF_MAX_MS, ac: new AbortController(), - readyDeferred: Promise.withResolvers(), + readyWaiters: new Set(), closedDeferred: Promise.withResolvers(), isFinalized: false, timers: new Map(), @@ -921,9 +901,6 @@ export let createEndpointsRuntime = function createEndpointsRuntime( lastPileStates: [], prevFallbackActive: false, } - // Silence unobserved-rejection noise when nobody awaits ready(). - env.readyDeferred.promise.catch(() => {}) - let ctx = createEndpointsCtx({ localityEnabled: config.localityEnabled, preferPrimaryPile: config.preferPrimaryPile, diff --git a/packages/core/src/endpoints/endpoints-state.ts b/packages/core/src/endpoints/endpoints-state.ts index 6ba95c88..f57dd731 100644 --- a/packages/core/src/endpoints/endpoints-state.ts +++ b/packages/core/src/endpoints/endpoints-state.ts @@ -109,9 +109,7 @@ export type EndpointsCtx = { pinned: Map attempts: number - lastError: unknown roundInFlight: boolean - hasEverDiscovered: boolean selfLocation: string // Empty ⇒ the cluster is not in bridge mode ⇒ the pile filter is identity. @@ -131,9 +129,7 @@ export let createEndpointsCtx = function createEndpointsCtx(config?: { byNodeId: new Map(), pinned: new Map(), attempts: 0, - lastError: undefined, roundInFlight: false, - hasEverDiscovered: false, selfLocation: '', pileStates: [], config: { @@ -199,11 +195,12 @@ export type EndpointsEffect = | ({ type: 'endpoints.effect.timer.clear' } & TimerRef) // Retire-to-drain: keep the channel open, move it to the drain-watch set. | { type: 'endpoints.effect.retire_channel'; nodeId: bigint } - // Physically close and drop the channel. `store` scopes which materialized - // channel is dropped: 'pinned' closes only the pin (an invalidate must not - // tear down a discovered channel that shares the same nodeId); 'any' (default) - // closes whichever store holds it. - | { type: 'endpoints.effect.close_channel'; nodeId: bigint; store?: 'any' | 'pinned' } + // Retirement and pin invalidation close their own store; shutdown closes both. + | { + type: 'endpoints.effect.close_channel' + nodeId: bigint + store?: 'any' | 'discovered' | 'pinned' + } // Begin the graceful close drain: close idle channels now and wait for // in-flight streams to finish (bounded by the close deadline). Runtime-only. | { type: 'endpoints.effect.begin_close_drain' } @@ -415,7 +412,11 @@ let applyRound = function applyRound( // (A brief flap keeps the same address and is absorbed by retire-drain.) let newAddress = `${ep.host}:${ep.port}` if (existing.address !== newAddress) { - effects.push({ type: 'endpoints.effect.close_channel', nodeId: ep.nodeId }) + effects.push({ + type: 'endpoints.effect.close_channel', + nodeId: ep.nodeId, + store: 'discovered', + }) } // Refresh surface fields (location/pile/load/dial info can change). @@ -477,7 +478,6 @@ let applyRound = function applyRound( ctx.selfLocation = selfLocation ctx.pileStates = pileStates ctx.attempts = 0 - ctx.lastError = undefined ctx.roundInFlight = false rebuild(ctx, runtime) @@ -682,8 +682,6 @@ export let endpointsTransition = function endpointsTransition( switch (event.type) { case 'endpoints.discovery.round_succeeded': { if (event.endpoints.length === 0) return rejectEmptyRound(ctx, runtime) - let firstReady = !ctx.hasEverDiscovered - ctx.hasEverDiscovered = true let effects = applyRound( ctx, event.endpoints, @@ -696,12 +694,11 @@ export let endpointsTransition = function endpointsTransition( which: 'discovery_interval', }) effects.push({ type: 'endpoints.effect.timer.schedule', which: 'idle_sweep' }) - if (firstReady) runtime.emit({ type: 'endpoints.ready' }) + runtime.emit({ type: 'endpoints.ready' }) return { state: healthState(ctx), effects } } case 'endpoints.discovery.round_failed': { ctx.attempts += 1 - ctx.lastError = event.error ctx.roundInFlight = false runtime.emit({ type: 'endpoints.discovery_failed', @@ -754,7 +751,6 @@ export let endpointsTransition = function endpointsTransition( // Background failure is never terminal — keep serving the last // snapshot; the interval/backoff retries. ctx.attempts += 1 - ctx.lastError = event.error ctx.roundInFlight = false runtime.emit({ type: 'endpoints.discovery_failed', @@ -837,7 +833,13 @@ export let endpointsTransition = function endpointsTransition( reason: 'idle', }) return { - effects: [{ type: 'endpoints.effect.close_channel', nodeId: event.nodeId }], + effects: [ + { + type: 'endpoints.effect.close_channel', + nodeId: event.nodeId, + store: 'discovered', + }, + ], } } case 'endpoints.pin': diff --git a/packages/core/src/endpoints/endpoints.contract.test.ts b/packages/core/src/endpoints/endpoints.contract.test.ts index ff9c7048..57b4515f 100644 --- a/packages/core/src/endpoints/endpoints.contract.test.ts +++ b/packages/core/src/endpoints/endpoints.contract.test.ts @@ -250,6 +250,25 @@ test('direct-IO: pin then acquire the exact server-named node', async (tc) => { expect(conn.endpoint.nodeId).toBe(9n) }) +test.for([false, true])( + 'keeps pinned and discovered channels separate (pin first: %s)', + async (pinFirst, tc) => { + // Discovery and pin publication order must be controlled independently of network timing. + await using h = setup([endpoint(1)]) + await h.pool.ready(tc.signal) + h.pool.pin(1n, 'direct-node', 2137) + await settle() + let direct = () => h.pool.acquireNode(1n, { hard: true }) + let first = pinFirst ? direct() : h.pool.acquire(1n) + let second = pinFirst ? h.pool.acquire(1n) : direct() + expect(first.endpoint.address).toBe(pinFirst ? 'direct-node:2137' : 'node-1:2136') + expect(second.endpoint.address).toBe(pinFirst ? 'node-1:2136' : 'direct-node:2137') + await h.pool.close() + expect(h.connections.materialized).toHaveLength(2) + expect(h.connections.materialized.every((connection) => connection.closed)).toBe(true) + } +) + test('publishes an explicit pin while initial discovery is still pending', async () => { let discovery = makeFakeDiscovery() discovery.hang() @@ -265,6 +284,55 @@ test('direct-IO: a hard-pin to an absent node throws', async (tc) => { expect(() => h.pool.acquireNode(99n, { hard: true })).toThrow(EndpointsUnavailableError) }) +test('invalidating a pin preserves the discovered channel for the same node', async (tc) => { + await using h = setup([endpoint(1)]) + await h.pool.ready(tc.signal) + let discovered = h.pool.acquire(1n) + h.pool.pin(1n, 'direct-node', 2137) + await settle() + h.pool.acquireNode(1n, { hard: true }) + h.pool.invalidate(1n) + await settle() + expect(h.pool.acquire(1n)).toBe(discovered) + expect(h.connections.materialized.map((connection) => connection.closed)).toEqual([false, true]) +}) + +test('retiring discovery preserves the pinned channel for the same node', async (tc) => { + await using h = setup([endpoint(1), endpoint(2)], { retiredGraceMs: 0 }) + await h.pool.ready(tc.signal) + h.pool.acquire(1n) + h.pool.pin(1n, 'direct-node', 2137) + await settle() + let pinned = h.pool.acquireNode(1n, { hard: true }) + h.discovery.push(discoveryResult([endpoint(2)])) + h.pool.forceRediscovery() + await settle() + h.machine.dispatch({ type: 'endpoints.timer.idle_sweep' }) + await settle() + expect(h.pool.acquireNode(1n, { hard: true })).toBe(pinned) + expect(h.connections.materialized.map((connection) => connection.closed)).toEqual([true, false]) +}) + +test('close waits for a busy pin when its discovered channel was never opened', async (tc) => { + await using h = setup([endpoint(1)]) + await h.pool.ready(tc.signal) + h.pool.pin(1n, 'direct-node', 2137) + await settle() + h.pool.acquireNode(1n, { hard: true }) + h.pool.callStarted(1n) + let closed = false + let closing = h.pool.close().then(() => { + closed = true + return closed + }) + await settle() + expect(closed).toBe(false) + expect(h.connections.materialized[0]!.closed).toBe(false) + h.pool.callEnded(1n) + await closing + expect(h.connections.materialized[0]!.closed).toBe(true) +}) + test('releasing the last pin makes an undiscovered node unreachable', async (tc) => { await using h = setup([endpoint(1)]) await h.pool.ready(tc.signal) @@ -447,6 +515,8 @@ test('a non-retryable initial failure rejects ready with the cause', async (tc) // A non-retryable failure is terminal; ready() rejects with the real cause, // not a generic 'Endpoints closed'/'finalized'. await expect(h.pool.ready(tc.signal)).rejects.toBe(boom) + await settle() + await expect(h.pool.ready(tc.signal)).rejects.toBe(boom) }) test('a throwing onDiscovery hook does not break the pool', async (tc) => { @@ -566,3 +636,49 @@ test('mapDiscoveryResult leaves pile states empty for a non-bridge cluster', () expect(dto.pileStates).toHaveLength(0) expect(dto.endpoints[0]!.bridgePileName).toBe('') }) + +test('keeps a live channel through repeated retirement and revival', async (tc) => { + // Fake discovery changes the roster without interrupting the channel's active RPC. + let discovery = makeFakeDiscovery() + await using h = setup([endpoint(1), endpoint(2)], { discovery }) + await h.pool.ready(tc.signal) + h.pool.acquire(2n) + let connection = h.connections.byNode(2n)! + connection.driveState(connectivityState.READY) + h.pool.callStarted(2n) + try { + for (let endpoints of [[endpoint(1)], [endpoint(1), endpoint(2)], [endpoint(1)]]) { + discovery.push(discoveryResult(endpoints)) + h.pool.forceRediscovery() + // oxlint-disable-next-line no-await-in-loop + await settle() + expect(connection.closed).toBe(false) + expect(h.pool.acquire(2n)).toBe(connection) + } + expect(h.connections.factoryCalls()).toBe(1) + } finally { + h.pool.callEnded(2n) + } +}) + +test('rejects ready after a previously ready pool closes', async (tc) => { + await using h = setup([endpoint(1)]) + await h.pool.ready(tc.signal) + await h.pool.close() + await expect(h.pool.ready(tc.signal)).rejects.toThrow(/closed/i) +}) + +test('rejects ready while a live call keeps the pool closing', async (tc) => { + await using h = setup([endpoint(1)]) + await h.pool.ready(tc.signal) + h.pool.acquire(1n) + h.pool.callStarted(1n) + let closing = h.pool.close() + try { + await settle() + await expect(h.pool.ready(tc.signal)).rejects.toThrow(/clos/i) + } finally { + h.pool.callEnded(1n) + await closing + } +}) diff --git a/packages/core/src/endpoints/endpoints.lifecycle.test.ts b/packages/core/src/endpoints/endpoints.lifecycle.test.ts index 9a53e129..c0bb7e44 100644 --- a/packages/core/src/endpoints/endpoints.lifecycle.test.ts +++ b/packages/core/src/endpoints/endpoints.lifecycle.test.ts @@ -1,10 +1,12 @@ import { getEventListeners } from 'node:events' import { memoryUsage } from 'node:process' +import { setImmediate } from 'node:timers/promises' import { expect, test } from 'vitest' import { EndpointsUnavailableError } from '../errors.ts' import { + capture, discoveryResult, endpoint, makeEndpointPool, @@ -61,6 +63,71 @@ test('ready(signal) leaves no abort listeners on a shared signal', async (tc) => expect(getEventListeners(signal, 'abort')).toHaveLength(0) }) +test('cancelled ready calls release their reasons before discovery completes', async () => { + if (gc === undefined) throw new Error('This test requires --expose-gc') + // A blocked discovery deterministically keeps readiness pending through every cancellation. + let discovery = makeFakeDiscovery() + discovery.hang() + await using h = makeEndpointPool({ discovery }) + await discovery.waitForRound(1) + + let cancelReady = async () => { + let controller = new AbortController() + let reason = { cancelled: true } + let reference = new WeakRef(reason) + let pending = h.pool.ready(controller.signal) + controller.abort(reason) + try { + await pending + throw new Error('Cancelled ready resolved') + } catch (error) { + if (error !== reason) throw error + } + return reference + } + let reasons: WeakRef[] = [] + for (let i = 0; i < 100; i++) { + // oxlint-disable-next-line no-await-in-loop + reasons.push(await cancelReady()) + } + for (let i = 0; i < 3; i++) { + // oxlint-disable-next-line no-await-in-loop + await setImmediate() + gc() + } + await setImmediate() + + expect(h.machine.state).toBe('discovering') + expect(discovery.lastSignal()!.aborted).toBe(false) + expect(reasons.filter((reference) => reference.deref() !== undefined)).toHaveLength(0) +}) + +test('cancelling one ready waiter leaves the others waiting for published readiness', async (tc) => { + using readyEvents = capture('ydb:driver.ready') + await using h = spinUp() + let controller = new AbortController() + let reason = new Error('Caller cancelled') + let cancelled = h.pool.ready(controller.signal).catch((error: unknown) => error) + let remaining = Promise.all([h.pool.ready(tc.signal), h.pool.ready()]).then( + () => readyEvents.events.length + ) + controller.abort(reason) + + expect(await cancelled).toBe(reason) + expect(await remaining).toBe(1) + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0) + expect(h.pool.acquire().endpoint.nodeId).toBeDefined() +}) + +test('rejects ready when close starts before its continuation', async (tc) => { + await using h = spinUp() + await h.pool.ready(tc.signal) + let ready = h.pool.ready(tc.signal) + let closed = h.pool.close() + await expect(ready).rejects.toThrow(/closed/i) + await closed +}) + test('close closes every materialized channel and leaves none behind', async (tc) => { let connections = makeFakeConnectionFactory() let discovery = makeFakeDiscovery() diff --git a/packages/core/src/endpoints/snapshot.test.ts b/packages/core/src/endpoints/snapshot.test.ts index 479524d1..5a3a3f8b 100644 --- a/packages/core/src/endpoints/snapshot.test.ts +++ b/packages/core/src/endpoints/snapshot.test.ts @@ -38,9 +38,7 @@ let ctxOf = function ctxOf( byNodeId: new Map(entries.map((e) => [e.nodeId, e])), pinned: new Map((over.pinned ?? []).map((e) => [e.nodeId, e])), attempts: 0, - lastError: undefined, roundInFlight: false, - hasEverDiscovered: true, selfLocation: over.selfLocation ?? 'A', pileStates: over.pileStates ?? [], config: { diff --git a/packages/fsm/src/queue.test.ts b/packages/fsm/src/queue.test.ts index 5591599b..ad2e9285 100644 --- a/packages/fsm/src/queue.test.ts +++ b/packages/fsm/src/queue.test.ts @@ -1,6 +1,21 @@ +import { setTimeout as sleep } from 'node:timers/promises' + import { expect, test } from 'vitest' import { AsyncPriorityQueue, AsyncQueue } from './queue.ts' +test('cancellation after resume leaves the buffered item available to the next take', async () => { + using queue = new AsyncQueue() + let controller = new AbortController() + let reason = new Error('Cancelled before dequeue') + queue.pause() + queue.push(1) + let pending = queue.take(controller.signal) + queue.resume() + controller.abort(reason) + await expect(pending).rejects.toBe(reason) + expect(await queue.take()).toEqual({ done: false, value: 1 }) +}) + test('processes priority items in descending order', async () => { let queue = new AsyncPriorityQueue() @@ -547,3 +562,34 @@ test('take interleaves with the iterator without losing items', async () => { expect(await queue.take()).toEqual({ value: 'a', done: false }) expect((await iterator.next()).value).toBe('b') }) + +test('releases a yielded payload while the iterator waits for its next item', async (tc) => { + using queue = new AsyncQueue() + let iterator = queue[Symbol.asyncIterator]() + let collected = false + let registry = new FinalizationRegistry(() => { + collected = true + }) + let consumePayload = async () => { + let payload = new Uint8Array(8 * 1024 * 1024) + registry.register(payload.buffer, undefined) + queue.push(payload) + await iterator.next() + } + await consumePayload() + let pending = iterator.next() + + // The registry callback runs between GC rounds; neither queue nor iterator is discarded. + // oxlint-disable-next-line no-unmodified-loop-condition + for (let attempt = 0; attempt < 40 && !collected; attempt++) { + globalThis.gc!() + // oxlint-disable-next-line no-await-in-loop + await sleep(10, undefined, { signal: tc.signal }) + } + expect(collected).toBe(true) + let next = new Uint8Array([42]) + queue.push(next) + await expect(pending).resolves.toEqual({ value: next, done: false }) + queue.close() + await expect(iterator.next()).resolves.toMatchObject({ done: true }) +}) diff --git a/packages/fsm/src/queue.ts b/packages/fsm/src/queue.ts index 31f6a484..84fdf554 100644 --- a/packages/fsm/src/queue.ts +++ b/packages/fsm/src/queue.ts @@ -122,7 +122,7 @@ export abstract class AbstractAsyncQueue implements AsyncIterable, Disp async *[Symbol.asyncIterator](): AsyncIterator { while (true) { // oxlint-disable-next-line no-await-in-loop - let next = await this.#next() + let next: IteratorResult | undefined = await this.#next() if (next.done) { if (this.#failure) { throw this.#failure.error @@ -132,6 +132,8 @@ export abstract class AbstractAsyncQueue implements AsyncIterable, Disp } yield next.value + // Do not retain the delivered payload while waiting for another item. + next = undefined } } @@ -143,9 +145,8 @@ export abstract class AbstractAsyncQueue implements AsyncIterable, Disp // thus swallow) an item. For a bounded wait compose the deadline at the call site: // linkSignals(signal, AbortSignal.timeout(ms)). async take(signal?: AbortSignal): Promise> { - signal?.throwIfAborted() - for (;;) { + signal?.throwIfAborted() if (this.#destroyed) { return { value: undefined as never, done: true } } diff --git a/packages/fsm/vitest.config.ts b/packages/fsm/vitest.config.ts index d22783e2..97c3f512 100644 --- a/packages/fsm/vitest.config.ts +++ b/packages/fsm/vitest.config.ts @@ -5,5 +5,6 @@ export default defineProject({ name: 'fsm', include: ['src/**/*.test.ts', 'tests/**/*.test.ts'], environment: 'node', + execArgv: ['--expose-gc'], }, }) diff --git a/packages/topic/ARCHITECTURE.md b/packages/topic/ARCHITECTURE.md index 2d1a561a..4296089f 100644 --- a/packages/topic/ARCHITECTURE.md +++ b/packages/topic/ARCHITECTURE.md @@ -1,202 +1,105 @@ # @ydbjs/topic — state machine maps -Contributor documentation. The reader and writer are each built from two finite state -machines on top of `@ydbjs/fsm`, plus a thin public facade. This file is the map of -every machine: states, transitions, timers, and the invariants the design rests on. +The reader and writer each have one protocol state machine, an I/O runtime, a stream transport and a public facade. The transition tables map protocol events to state changes, effects and observable outputs. -The transition tables below mirror the exhaustive `switch` dispatch in the `*-state.ts` -files one-to-one — every `(state, event)` pair is either a table row or falls into the -`(everything else)` row. **When you change a dispatch, update the matching table and -diagram in the same commit.** - -## The two-FSM pipeline +## Ownership ```mermaid flowchart LR - subgraph facade [Facade — writer.ts / reader.ts] - API[public API
write/flush/close · read/commit/close] - end - subgraph domain [Domain FSM — writer-state.ts / reader-state.ts] - D[pure transition
ctx mutations, effects, outputs] - end - subgraph transport [Transport FSM — transport-state.ts] - T[one gRPC stream lifecycle
classify server frames] - end - G[(gRPC StreamWrite /
StreamRead)] - - API -- events --> D - D -- outputs (AsyncQueue) --> API - D -- effects --> R[runtime
writer-runtime.ts / reader-runtime.ts] + API[Public facade
write/flush/close · read/commit/close] + D[Protocol FSM
writer-state.ts / reader-state.ts] + R[Runtime
timers and I/O effects] + T[Transport
one physical stream] + G[(gRPC StreamWrite / StreamRead)] + API -- commands --> D + D -- outputs --> API + D -- effects --> R R -- events --> D R -- connect/send --> T - T -- outputs --> R + T -- stream events --> R T <--> G ``` -- The **domain FSM** is pure and synchronous: it mutates `ctx` in place and returns the - next state plus a list of effects. All I/O lives in the **runtime**, which executes - effects (connect, send, timers) and feeds results back as events. -- The **transport FSM** owns exactly one physical stream at a time: it classifies raw - server frames into typed facts and reports stream life/death. Reconnect policy lives - in the domain FSM, never in the transport. -- Outputs flow to the facade through an `AsyncQueue` with drain-then-throw semantics: - on a fatal error the queue delivers already-emitted facts (acks, commit confirmations) - first, then throws the reason. -- Convention: the domain FSMs log deliberately-ignored `(state, event)` pairs via - `ignored()`; the transport FSMs drop unhandled events silently — they are pure - classifiers, and every meaningful fact already has an explicit case. - ---- +- The facade validates synchronous API calls and owns application promises, callbacks and delivered messages. +- The protocol FSM owns sequence numbers, partition grants, unconfirmed work and reconnect policy. Transitions mutate their context and return effects; they do not perform I/O. +- The runtime executes effects and feeds timer and transport events into the FSM. +- The transport owns the physical stream and its outgoing queue. It reports received frames and disconnects without keeping a second protocol state machine. +- Machine outputs drain already-emitted facts before a terminal error reaches the facade. A runtime fault also releases timers, transport resources and retained messages. + +Update the corresponding transition table when changing a protocol transition. ## Writer FSM (`writer-state.ts`) -- `idle` — created, not started; accepts writes into the buffer before connecting -- `connecting` — one stream attempt in flight, `start_timeout` watchdog armed -- `ready` — live session; pumps buffer → inflight, handles acks/token -- `reconnecting` — backing off between attempts (`retry_backoff` armed; `recovery_window` armed only if `recoveryWindowMs` is finite) -- `closing` — graceful drain gate, bounded by `graceful_timeout`; keeps reconnecting to finish the drain -- `closed` — graceful/destroyed terminal (final) -- `errored` — fatal terminal (final) +The connection has six states: `idle`, `connecting`, `ready`, `reconnecting`, `closed` and `errored`. `closeRequested` records the independent request to drain and close. Closing preserves the current connection phase, so a reconnect still requires InitResponse before any writes are sent. ```mermaid stateDiagram-v2 [*] --> idle idle --> connecting: writer.start - idle --> closed: writer.close connecting --> ready: stream.init_response - connecting --> reconnecting: timer.start_timeout, stream.disconnected [retryable] - connecting --> errored: stream.disconnected [fatal], timer.recovery_window - connecting --> closing: writer.close - ready --> reconnecting: stream.disconnected [retryable] - ready --> errored: stream.disconnected [fatal] - ready --> closing: writer.close - reconnecting --> ready: stream.init_response - reconnecting --> connecting: timer.retry_backoff - reconnecting --> errored: timer.recovery_window - reconnecting --> closing: writer.close - closing --> closed: drained (write_response, init_response, graceful_timeout) - closing --> errored: timer.graceful_timeout [pending], stream.disconnected [fatal] + connecting --> reconnecting: start_timeout / retryable disconnect + reconnecting --> connecting: retry_backoff + reconnecting --> ready: late init_response + ready --> reconnecting: retryable disconnect + connecting --> errored: fatal error / recovery deadline + reconnecting --> errored: fatal error / recovery deadline + ready --> errored: fatal error + idle --> closed: empty close / destroy + connecting --> closed: drained close / destroy + reconnecting --> closed: drained close / destroy + ready --> closed: drained close / destroy closed --> [*] errored --> [*] - note right of closed - writer.destroy from any non-terminal state lands here. - writer.close with an already-empty window skips closing - and terminates directly in closed. - end note ``` -Helper resolution: `toReady` → `ready`; `toReconnecting` → `reconnecting`; `toClosing` → -`closing` (or straight to `closed` via `closeWhenDrained` if the window is already -empty); `terminate(s)` → `s` with `final:{reason}`, emits `writer.closed` (plus -`writer.error` when `errored`), frees the message window, effects `[transport.close, -finalize]`. `pump` keeps the state — effect `send.write_request(batch)` when sendable, -self-dispatches `writer.pump` while more can be sent. `connectEffects` = -`[transport.connect(getLastSeqNo=!hasEverConnected), schedule start_timeout]`. - -| state | event | → next | key effects / outputs / timers | -| ---------------- | -------------------------------------------------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| any non-terminal | `writer.destroy` | closed | terminate: emit `writer.closed{reason ?? 'Writer destroyed'}`; release window | -| idle | `writer.start` | connecting | `transport.connect(getLastSeqNo=true)`, schedule `start_timeout` | -| idle | `writer.write` | same | enqueue (infers seqNoMode on first message; manual seqNo raises `lastSeqNo` HWM) | -| idle | `writer.close` | closed | terminate('Writer closed before start') | -| idle | (everything else) | same | ignored (logged) | -| connecting | `writer.write` | same | enqueue | -| connecting | `writer.flush` | same | set `flushRequested`; emit `writer.flushed` if window empty | -| connecting | `writer.stream.init_response` | ready / errored | toReady: codec gate first — a non-empty `supportedCodecs` excluding the session codec terminates `errored` before anything reaches the wire; else `attempts=0`; applyInit (recover seqNo HWM once, dedup server-persisted inflight → emit `writer.acknowledgments{skipped}`; emit `writer.session`); resolve flush if drained; dispatch `writer.pump`; clear `start_timeout`/`retry_backoff`/`recovery_window`; schedule `flush_tick`+`update_token` | -| connecting | `writer.timer.start_timeout` | reconnecting | toReconnecting(no error) | -| connecting | `writer.stream.disconnected` [retryable] | reconnecting | toReconnecting(error): record `lastError`; emit `writer.reconnecting{attempt,error}`; schedule `retry_backoff` (+`recovery_window` if finite) | -| connecting | `writer.stream.disconnected` [fatal] | errored | terminate(error): emit `writer.error`+`writer.closed` | -| connecting | `writer.timer.recovery_window` | errored | terminate(`lastError ?? 'Writer recovery window expired'`) | -| connecting | `writer.close` | closing / closed | toClosing: dispatch `writer.pump`; schedule `graceful_timeout` (start_timeout/retry_backoff left armed) | -| connecting | (everything else) | same | ignored (logged) | -| ready | `writer.write` | same | enqueue; dispatch `writer.pump` | -| ready | `writer.pump` / `writer.timer.flush_tick` | same | pump: form batch (buffer→inflight, assign auto seqNos); `send.write_request`; re-dispatch while sendable | -| ready | `writer.stream.write_response` | same | acknowledge acked prefix → emit `writer.acknowledgments{freedBytes}`; emit `writer.flushed` if flush pending and drained; dispatch `writer.pump` | -| ready | `writer.flush` | same | requestFlush (emit `writer.flushed` if drained, else dispatch `writer.pump`) | -| ready | `writer.timer.update_token` | same | `send.update_token` | -| ready | `writer.stream.token_response` | same | consumed, no-op | -| ready | `writer.stream.disconnected` [retryable] | reconnecting | toReconnecting(error) | -| ready | `writer.stream.disconnected` [fatal] | errored | terminate(error) | -| ready | `writer.close` | closing / closed | toClosing | -| ready | (everything else) | same | ignored (logged) | -| reconnecting | `writer.write` | same | enqueue | -| reconnecting | `writer.flush` | same | requestFlush | -| reconnecting | `writer.stream.init_response` | ready | toReady (late init from a still-open stream is honored) | -| reconnecting | `writer.timer.retry_backoff` | connecting | `attempts += 1`; connectEffects | -| reconnecting | `writer.timer.recovery_window` | errored | terminate(`lastError ?? 'Writer recovery window expired'`) | -| reconnecting | `writer.stream.disconnected` | same | record `lastError` if present; stay (already backing off) | -| reconnecting | `writer.close` | closing / closed | toClosing | -| reconnecting | (everything else) | same | ignored (logged) | -| closing | `writer.stream.write_response` | closed [drained] / same | acknowledge → emits as in ready; closeWhenDrained: terminate('Writer closed') when window empty, else dispatch `writer.pump` | -| closing | `writer.pump` / `flush_tick` [`!hasEverConnected`] | same | no-op guard: never assign auto seqNos before the HWM is recovered (prevents dedup data loss when close() races the first init) | -| closing | `writer.pump` / `flush_tick` [`hasEverConnected`] | same | pump | -| closing | `writer.flush` | same | requestFlush | -| closing | `writer.stream.init_response` | closed [drained] / same | applyInit; resolve flush; closeWhenDrained; clear `start_timeout` | -| closing | `writer.timer.retry_backoff` | same | connectEffects — keeps draining over a fresh stream | -| closing | `writer.timer.graceful_timeout` [not drained] | errored | terminate('Graceful shutdown timed out with undelivered messages') — close() rejects instead of dropping writes | -| closing | `writer.timer.graceful_timeout` [drained] | closed | closeWhenDrained → terminate('Writer closed') | -| closing | `writer.timer.start_timeout` | same | clear `start_timeout`, schedule `retry_backoff` | -| closing | `writer.stream.disconnected` [retryable] | same | clear `start_timeout`, schedule `retry_backoff` | -| closing | `writer.stream.disconnected` [fatal] | errored | terminate(error) | -| closing | (everything else) | same | ignored (logged) | -| closed / errored | (everything, incl. `writer.destroy`) | same | ignored (logged) | - -Retryable classification (`isRetryableWriterError`): a clean end (no error) is -retryable; payload-too-large (`ClientError RESOURCE_EXHAUSTED` + `/larger than/i`) is -always fatal; `SCHEME_ERROR` is retryable only with `retryOnSchemeError`; otherwise -`isRetryableStreamError || isRetryableError(err, idempotent=true)` — writes are -idempotent (producerId+seqNo dedup), so the conditionally-retryable YDB statuses retry. - -## Writer transport FSM (`writer/transport-state.ts`) - -- `idle` — created, no stream yet -- `connecting` — `open_stream` issued, awaiting init (one physical streamWrite lifecycle) -- `ready` — init seen; forwarding classified stream facts as outputs -- `disconnected` — stream ended/errored; awaiting `transport.connect` (reopen) or `transport.close` -- `closed` — terminal (final) +`messages` retains accepted, unacknowledged messages in order. `inflightCount` separates the sent prefix from the unsent suffix. Acknowledgments remove a prefix; reconnect removes the server-confirmed prefix and makes the remainder eligible for resend. Automatic sequence numbers are assigned only on sending, after the initial server watermark has been recovered. Manual numbers are validated by the facade and retained unchanged. -```mermaid -stateDiagram-v2 - [*] --> idle - idle --> connecting: transport.connect - idle --> closed: transport.close - connecting --> ready: transport.init - connecting --> connecting: transport.connect (reopen) - connecting --> disconnected: transport.ended, transport.error - connecting --> closed: transport.close - ready --> connecting: transport.connect (reopen) - ready --> disconnected: transport.ended, transport.error - ready --> closed: transport.close - disconnected --> connecting: transport.connect - disconnected --> closed: transport.close - closed --> [*] - note right of closed - transport.destroy from any non-closed state lands here. - end note -``` +The facade counts unacknowledged compressed payload bytes for `maxBufferBytes`. The FSM caches the compressed payload bytes in the unsent suffix for constant-time batch readiness checks. Batching targets 48 MiB of compressed payload; protocol framing and metadata do not reduce this payload budget. Diagnostics report the same compressed payload bytes. + +A batch is full when it fills the available `maxInflightCount` slots, reaches the byte cap, or the next queued message cannot fit within that cap. Full batches send immediately. A partial batch waits for `flushIntervalMs`, unless an explicit `flush()` or `close()` requires draining it. `batchDue` records that the timer expired while messages were waiting: if in-flight messages occupy the window, their ACK must release the overdue batch without another timer delay. All messages retained across a reconnect become immediately eligible once InitResponse arrives, including partial batches that had not reached the wire. + +`flushes` maps each captured last message to the latest flush ID waiting for that boundary. Several calls without intervening writes share one boundary; later writes cannot extend an earlier flush. `closeRequested` stops admission and keeps reconnecting until all accepted messages are acknowledged. The graceful deadline bounds that drain instead of the normal recovery window. ACK outputs precede flush completion, which precedes terminal close. + +| State | Event | Result | +| -------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Any live state | `writer.destroy` | Emit closed and release all resources. | +| Any live state | `writer.close` | Record close intent once; close immediately if drained, otherwise arm the graceful deadline and continue the current connection lifecycle. | +| Any live state | `writer.write` | Enqueue while accepting writes; trigger the batch planner when ready. Ignore writes after close intent. | +| Any live state | `writer.flush` | Capture the last queued message; complete if empty, otherwise request a pump. | +| Any live state | `writer.timer.flush_tick` | Mark a waiting batch due; pump only when ready. | +| Any live state | `writer.timer.graceful_timeout` | If closing, terminate with an error when messages remain; otherwise finish the drain. Ignore when no close was requested. | +| `idle` | `writer.start` | Connect and arm the start watchdog. | +| `connecting` / `reconnecting` | `writer.stream.init_response` | Validate the codec, recover sequence numbers and deduplicate; finish a completed drain or enter ready. Clear connection/recovery timers and arm flush/token timers. | +| `connecting` | `writer.timer.start_timeout` | Enter reconnect backoff. | +| `reconnecting` | `writer.timer.retry_backoff` | Increment attempts, reconnect and arm the start watchdog. | +| `connecting` / `ready` | Retryable disconnect | Enter reconnect backoff; retain accepted messages. | +| `reconnecting` | Retryable disconnect | Retain the error while the existing backoff remains armed. | +| Any connected/reconnecting state | Fatal disconnect | Emit error then closed, releasing all resources. | +| `connecting` / `reconnecting` | `writer.timer.recovery_window` | Fail unless the graceful-close deadline has taken over. | +| `ready` | `writer.pump` | Plan and send an eligible batch; continue while more batches are sendable. | +| `ready` | `writer.stream.write_response` | Release acknowledged messages, emit acknowledgments, complete acknowledged flush boundaries and close when drained, and pump the remaining queue. | +| `ready` | `writer.timer.update_token` | Request a token refresh on the current stream. | +| `ready` | `writer.stream.token_response` | No protocol-state change. | +| Any state | Other events | Ignore; terminal states accept no further work. | + +Reader and writer use the same topic retry policy. A clean stream end and a server gRPC deadline reconnect. `SCHEME_ERROR` retries only with `retryOnSchemeError`. Producer/seqNo deduplication and consumer offsets make replay safe for conditionally retryable YDB statuses. A deterministic gRPC frame-size rejection is terminal. User cancellation stays separate: the transport checks its aborted signal before reporting a disconnection. -| state | event | → next | key effects / outputs | -| ------------------ | ------------------------------------- | ------------ | ----------------------------------------------------------------------- | -| any non-closed | `transport.destroy` | closed | `close_stream`, `finalize(reason ?? 'Transport destroyed')` | -| idle | `transport.connect` | connecting | `open_stream` | -| idle | `transport.close` | closed | `finalize('Transport closed')` (nothing open — no `close_stream`) | -| connecting / ready | `transport.connect` | connecting | `open_stream` (reopen; disposes the previous stream first) | -| connecting / ready | `transport.init` | ready / same | emit `transport.stream.init_response{sessionId,lastSeqNo,partitionId?}` | -| connecting / ready | `transport.write` | same | emit `transport.stream.write_response{acks}` | -| connecting / ready | `transport.token` | same | emit `transport.stream.token_response` | -| connecting / ready | `transport.ended` / `transport.error` | disconnected | emit `transport.stream.disconnected{error?}`; `close_stream` | -| connecting / ready | `transport.close` | closed | `close_stream`, `finalize('Transport closed')` | -| disconnected | `transport.connect` | connecting | `open_stream` | -| disconnected | `transport.close` | closed | `finalize('Transport closed')` | -| any | (everything else) | same | silently dropped (pure classifier) | - ---- +The runtime finalizer also runs when the machine signal aborts after an internal fault, releasing transport, timers and the message window even when no terminal transition completes. + +### Flush request correlation + +`requestId` numbers `flush()` calls within one writer. It is internal, is never sent to YDB, and is unrelated to message `seqNo`. The facade registers a promise before dispatching the request; the FSM completes it through an asynchronous output queue. + +Consider `first = writer.flush(); writer.write(payload); second = writer.flush()` on an empty writer. The FSM can emit the first completion before the write, while the facade consumes that completion after the second call has already registered. Resolving all current waiters would incorrectly complete `second` before the payload's ACK. + +A completion carries the highest flush ID attached to the acknowledged boundary message. The facade resolves only IDs up to that value. Several calls waiting on the same message boundary may share one completion, but an old queued completion cannot resolve a newer call. Automatic message seqNos cannot identify these calls because buffered messages are numbered only when sent. ## Reader FSM (`reader-state.ts`) - `idle` — initial; only `reader.start` / `reader.close` are meaningful -- `connecting` — stream being opened; `start_timeout` armed -- `ready` — live stream. No seqNo recovery (unlike the writer) — the server re-sends `start_partition` per partition, and commit reconcile happens there +- `connecting` — waiting for InitResponse; the connection watchdog is active +- `waiting-credit` — initialized stream whose retained responses fill the read window; partition control, commits and token refresh remain active +- `ready` — initialized stream with its first read credit granted. No seqNo recovery (unlike the writer) — the server re-sends `start_partition` per partition, and commit reconcile happens there - `reconnecting` — backoff between attempts; `retry_backoff` armed, `recovery_window` only when finite - `closing` — `close()` drain of pending work (pending commits / stopping-graceful partitions), bounded by `graceful_timeout` - `closed` — terminal (clean close or destroy) @@ -207,12 +110,19 @@ stateDiagram-v2 [*] --> idle idle --> connecting: reader.start idle --> closed: reader.close - connecting --> ready: stream.init_response - reconnecting --> ready: stream.init_response + connecting --> ready: stream.init_response [buffer space] + reconnecting --> ready: stream.init_response [buffer space] + connecting --> waiting-credit: stream.init_response [buffer full] + reconnecting --> waiting-credit: stream.init_response [buffer full] + waiting-credit --> ready: reader.read_release [buffer space] connecting --> reconnecting: stream.disconnected [retryable], timer.start_timeout reconnecting --> connecting: timer.retry_backoff connecting --> errored: stream.disconnected [fatal], timer.recovery_window reconnecting --> errored: stream.disconnected [fatal], timer.recovery_window + waiting-credit --> reconnecting: stream.disconnected [retryable] + waiting-credit --> errored: stream.disconnected [fatal] + waiting-credit --> closing: reader.close [pending work] + waiting-credit --> closed: reader.close [no pending work] ready --> reconnecting: stream.disconnected [retryable] ready --> errored: stream.disconnected [fatal] connecting --> closing: reader.close [pending work] @@ -240,7 +150,7 @@ mapping lives in `ctx.sessionIndex` and is cleared on every reconnect): ```mermaid stateDiagram-v2 - [*] --> active: stream.start_partition (grantId++, ackPending) + [*] --> active: stream.start_partition (new session object, ackPending) active --> active: partition.start_ready → send start_response,\nre-send buffered commits (ackPending=false) active --> stopping_graceful: stop_partition [graceful] (stop hook + timer;\nstill delivers and commits) active --> stopped: stop_partition [force] @@ -257,30 +167,47 @@ stateDiagram-v2 The graceful stop is the protocol's soft-stop window: the entry keeps delivering buffered data and accepting commits while the async `onPartitionSessionStop` hook runs (effect `partition.stop_hook` → event `partition.stop_ready`, guarded by -`grantId` like the start handshake); the stop response goes out only once the hook +the session object identity like the start handshake); the stop response goes out only once the hook completed AND the pending commits drained — bounded by the per-partition timeout. Helper resolution: `terminate(s, reason)` → `s` with `final:{reason}`; emits `reader.error` (when `errored`), `reader.commit.rejected` for every pending commit, -`reader.closed`; clears ctx; effects `transport.close`, clear all timers, `finalize`. -`toReady` → `ready`: `attempts=0`, clear `sessionIndex`, reset flow-control; emits -`reader.session`; schedules `update_token`, sends `read_request(maxBufferBytes)`, and -arms `partition_reassign_gc:key` for every partition holding pending commits. -`toReconnecting(err)` → `reconnecting`: clear `sessionIndex`; emits -`reader.reconnecting`; schedules `retry_backoff` (+`recovery_window` iff finite). +`reader.closed`; clears ctx and runs `finalize`, which owns timer and transport cleanup. +`initializeSession` clears connection timers and `sessionIndex`, retains `bufferedBytes`, emits the server session ID through `reader.session`, and schedules `update_token`. A positive initial `maxBufferBytes - bufferedBytes` grant enters `ready`; otherwise the reader enters `waiting-credit`. +It arms `partition_reassign_gc:key` for every retained partition, including those whose messages may be committed after reconnect. +`toReconnecting(err)` → `reconnecting`: clear `sessionIndex` and retain `bufferedBytes`; emits `reader.reconnecting` and schedules `retry_backoff` (+`recovery_window` iff finite). `toClosing` → `closing` iff pending work exists, else terminate(closed, 'Reader closed'). `advanceCommitted(entry, offset)` — the single point where the server-confirmed committed watermark moves: compacts claimed ranges, lifts `deliveredWatermark`, and emits `partition.committed{session}` so the observer sees every advance (commit acks, stop watermarks, partition status responses). A start-response `commitOffset` remains a requested watermark until server confirmation; pending ranges remain available for replay after reconnect. Status queries start after the start response and retry after one second when a response reports a lower watermark. Only one status request is outstanding at a time. +Read flow control uses the server's whole `ReadResponse.bytesSize`; payload lengths and individual message sizes never determine a refund. +Each partition owns one canonical `pendingRanges` set for unconfirmed commits. `commitWaiters` store only the target offset and promise ID. Server confirmation trims the range set and settles covered waiters; reconnect sends the remaining ranges after the partition start handshake. Application promise boundaries do not determine wire-range ownership. + +`bufferedBytes` counts responses not yet fully consumed, including responses retained across reconnects. + +The facade queues encoded messages with their partition-session object. Decoding happens only for the messages selected for the next yield, so a paused consumer does not inflate its entire compressed backlog. The owned carry array also covers a suspended iterator; hard/fatal shutdown clears its chunks and the queue. Clean shutdown retains the encoded tail and its codec until the tail is consumed or discarded. Transaction offsets survive an early clean close until the transaction finishes. + +Stopped partition entries with no outstanding obligations are forgotten after earlier queued commits have observed their final watermark. Ended partitions remain committable. Runtime faults use the same resource cleanup as protocol termination, including releasing partition state and callback references. +After the current stream receives its initial grant, releasing a response of `N` bytes returns exactly `N` bytes in one `ReadRequest`, even if the response is split across many application batches. +For example, a 10 MiB response delivered as 100 batches returns no credit for the first 99 batches and exactly 10 MiB after the last one. +The protocol permits an oversized response to overdraw the server's allowance; returning that response's full size repays the overdraw without a separate client credit counter. + +The initial grant is the only exception: InitResponse enters `waiting-credit` when retained responses fill the window; releasing enough bytes transitions to `ready` and grants `maxBufferBytes - bufferedBytes`. +Before InitResponse, releases only reduce retained bytes. Session IDs are emitted as diagnostics and are not stored as initialization flags. +After InitResponse, partition hooks, commits, status checks and token refresh continue through the same handler as `ready`, even while a full retained buffer withholds the first ReadRequest. + +The `ready` session-event rows also apply to `waiting-credit`; read releases there grant the initial window only when retained bytes leave room. + | state | event | → next | key effects / outputs / timers | | ------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | any non-terminal | `reader.destroy` | closed | terminate(closed, `reason ?? 'Reader destroyed'`) | +| any non-terminal | `reader.read_release` | same / ready | `bufferedBytes -= bytes` (floor 0); in `ready`, refund the complete response size; in `waiting-credit`, a positive initial grant enters `ready`; before InitResponse, while reconnecting or closing, send nothing | | idle | `reader.start` | connecting | `transport.connect`, schedule `start_timeout` | | idle | `reader.close` | closed | terminate(closed, 'Reader closed before start') | | idle | (everything else) | same | ignored (logged) | -| connecting / reconnecting | `reader.stream.init_response` | ready | toReady | +| connecting / reconnecting | `reader.stream.init_response` | ready / waiting-credit | initializeSession | | connecting / reconnecting | `reader.commit` | same | recordCommit: buffer only (never sends outside ready); `commit.rejected` if partition unknown, `commit.resolved` if ranges already covered | | connecting / reconnecting | `reader.stream.disconnected` [fatal] | errored | terminate(errored, error) | | connecting / reconnecting | `reader.stream.disconnected` [retryable], `timer.start_timeout` | reconnecting | toReconnecting | @@ -291,18 +218,17 @@ stop watermarks, partition status responses). A start-response `commitOffset` re | connecting / reconnecting | `timer.partition_reassign_gc` | same | gcPartition: reject pending commits, delete entry unless it is a granted-but-unacked current grant; no-op if live+acked | | connecting / reconnecting | `reader.close` | closing / closed | toClosing | | connecting / reconnecting | (everything else) | same | ignored (logged) | -| ready | `reader.stream.read_response` | same | charge `inFlightBytes += bytesSize`; stitch each message's `commitRangeStart` from `deliveredWatermark`; emit one `reader.messages{releaseBytes, groups}` (drops data of unknown/superseded/stopped/ended entries — `stopping-graceful` still delivers; emitted even when all dropped so the credit is released) | -| ready | `reader.stream.start_partition` | same | upsert entry (key = topicPath+partitionId): →`active`, new `grantId`, `ackPending=true`, fresh session id (old id dropped from index); `deliveredWatermark`/claims restart at `committedOffset`; narrow pendings' ranges to it (resolve emptied); clear `partition_graceful_timeout:key`; emit `partition.started`; effect `partition.start_hook` (response deferred to `start_ready`) | +| ready | `reader.stream.read_response` | same | charge the whole response once: `bufferedBytes += bytesSize`; stitch each message's `commitRangeStart` from `deliveredWatermark`; emit one `reader.messages{releaseBytes, groups}` (drops data of unknown/superseded/stopped/ended entries — `stopping-graceful` still delivers; emitted even when all dropped so the credit is released) | +| ready | `reader.stream.start_partition` | same | upsert entry (key = topicPath+partitionId): →`active`, new session object, `ackPending=true`, fresh session id (old id dropped from index); `deliveredWatermark` restarts at `committedOffset`; trim `pendingRanges` and resolve covered waiters; clear `partition_graceful_timeout:key`; emit `partition.started`; effect `partition.start_hook` (response deferred to `start_ready`) | | ready | `reader.stream.stop_partition` [force] | same | advance committed (emit `partition.committed` if it moved) + resolve covered commits; entry→`stopped`, emit `partition.stopped(lost)`; clear `partition_graceful_timeout:key`; arm `partition_reassign_gc:key` if pendings remain | | ready | `reader.stream.stop_partition` [graceful] | same | advance committed (emit `partition.committed` if it moved) + resolve covered commits; entry→`stopping-graceful` (still delivers/commits); effect `partition.stop_hook`; arm `partition_graceful_timeout:key` | -| ready | `reader.stream.commit_response` | same | advance committed offsets (compact claimed ranges); emit `partition.committed` + `commit.resolved` for drained waiters; a drained `stopping-graceful` entry with `stopReady` →`stopped`: emit `partition.stopped(graceful)`, send `stop_response`, clear `partition_graceful_timeout:key` | +| ready | `reader.stream.commit_response` | same | advance committed offsets (trim `pendingRanges`); emit `partition.committed` + `commit.resolved` for drained waiters; a drained `stopping-graceful` entry with `stopReady` →`stopped`: emit `partition.stopped(graceful)`, send `stop_response`, clear `partition_graceful_timeout:key` | | ready / closing | `reader.stream.partition_status` | same / closed [closing, drained] | confirm server watermark like a commit response; schedule `partition_commit_status:key` only when the override is still unconfirmed; ignore retired sessions | | ready / closing | `timer.partition_commit_status` | same | request status only for a live grant with an unconfirmed override; regrant, disconnect and stop clear its timer | | ready | `reader.stream.end_partition` | same | entry→`ended`, session records child/adjacent partition ids, emit `partition.stopped(ended)`; no response; entry + session index kept so commits still reach the wire | | ready | `reader.commit` | same | recordCommit: clamp the facade's ranges to server committed, subtract claimed coverage (an overlap on the wire is session-fatal); fully covered → immediate `commit.resolved`; uncovered remainder on a `stopped` entry → immediate `commit.rejected` (the commit lost the race with the partition stop — never parked on the gc); else buffer pending + claim; `send.commit` only if the session is live on the current stream AND entry is `active`/`stopping-graceful`/`ended` AND `!ackPending`; `commit.rejected` when no entry exists | -| ready | `reader.partition.start_ready` | same | ackPartitionStart: no-op if grantId stale / not ackPending / not active / not indexed; else `ackPending=false`, `commitOffset` override sets the requested floor without confirming commits; effects `send.start_response`, clear `partition_reassign_gc:key`, re-send pending ranges above the requested floor, request partition status for an unconfirmed override | -| ready | `reader.partition.stop_ready` | same | ackPartitionStop: no-op if grantId stale / not `stopping-graceful`; else `stopReady=true`; when drained →`stopped`: emit `partition.stopped(graceful)`, send `stop_response` (only over the granting stream), clear `partition_graceful_timeout:key` | -| ready | `reader.read_release` | same | `inFlightBytes -= bytes`; accumulate credit; `read_request(credit)` once ≥ ceil(maxBufferBytes/5) | +| ready | `reader.partition.start_ready` | same | ackPartitionStart: no-op if session object superseded / not ackPending / not active / not indexed; else `ackPending=false`, `commitOffset` override sets the requested floor without confirming commits; effects `send.start_response`, clear `partition_reassign_gc:key`, re-send pending ranges above the requested floor, request partition status for an unconfirmed override | +| ready | `reader.partition.stop_ready` | same | ackPartitionStop: no-op if session object superseded / not `stopping-graceful`; else `stopReady=true`; when drained →`stopped`: emit `partition.stopped(graceful)`, send `stop_response` (only over the granting stream), clear `partition_graceful_timeout:key` | | ready | `timer.update_token` | same | `send.update_token` | | ready | `timer.partition_reassign_gc` / `timer.partition_graceful_timeout` | same | gcPartition / forceStopStalledGraceful (as above) | | ready | `reader.stream.disconnected` [fatal] | errored | terminate(errored, error) | @@ -319,62 +245,28 @@ stop watermarks, partition status responses). A start-response `commitOffset` re | closing | (everything else) | same | ignored (logged) | | closed / errored | (everything, incl. `reader.destroy`) | same | ignored (logged) | -## Reader transport FSM (`reader/transport-state.ts`) +## Transport lifecycle -Same five states as the writer transport. The only structural difference: after init, -every other server frame is forwarded verbatim as `transport.stream.message` — the -protobuf → domain classification happens in `reader-runtime.ts` (`classifyServerMessage`), -so the reader FSM never touches `serverMessage.case`. +`connect()` replaces the current physical stream and queues its InitRequest first. Stream input and cancellation belong to that connection attempt. A continuation after `driver.ready()`, token retrieval or stream iteration can affect only its original attempt; cancellation makes late results inert. -```mermaid -stateDiagram-v2 - [*] --> idle - idle --> connecting: transport.connect - idle --> closed: transport.close - connecting --> ready: transport.init - connecting --> connecting: transport.connect (reopen) - connecting --> disconnected: transport.ended, transport.error - connecting --> closed: transport.close - ready --> connecting: transport.connect (reopen) - ready --> disconnected: transport.ended, transport.error - ready --> closed: transport.close - disconnected --> connecting: transport.connect - disconnected --> closed: transport.close - closed --> [*] - note right of closed - transport.destroy from any non-closed state lands here. - end note -``` +A disconnect aborts the stream, discards unsent transport frames and emits a disconnected event. The protocol FSM retains the messages or commit ranges needed for recovery. Already received frames remain ordered before that disconnect. The transport event queue stays open for the next connection. + +`close()` and `destroy()` terminate the transport, abort the stream and discard its queues. A closed event queue prevents later `connect()` calls from reopening it. Token refresh keeps at most one request pending per stream until the server acknowledges it; an old refresh cannot send into or clear the pending request of a replacement stream. -| state | event | → next | key effects / outputs | -| ------------------ | ------------------------------------- | ------------ | ------------------------------------------------------------ | -| any non-closed | `transport.destroy` | closed | `close_stream`, `finalize(reason ?? 'Transport destroyed')` | -| idle | `transport.connect` | connecting | `open_stream` | -| idle | `transport.close` | closed | `finalize('Transport closed')` | -| connecting / ready | `transport.connect` | connecting | `open_stream` (reopen) | -| connecting / ready | `transport.init` | ready / same | emit `transport.stream.init_response{sessionId}` | -| connecting / ready | `transport.message` | same | emit `transport.stream.message{message}` (verbatim forward) | -| connecting / ready | `transport.ended` / `transport.error` | disconnected | emit `transport.stream.disconnected{error?}`; `close_stream` | -| connecting / ready | `transport.close` | closed | `close_stream`, `finalize('Transport closed')` | -| disconnected | `transport.connect` | connecting | `open_stream` | -| disconnected | `transport.close` | closed | `finalize('Transport closed')` | -| any | (everything else) | same | silently dropped (pure classifier) | - ---- +The writer classifies InitResponse, WriteResponse and UpdateTokenResponse into typed events. The reader classifies InitResponse and forwards the remaining frames for `reader-runtime.ts` to decode into protocol events. ## Invariants the maps rest on 1. **seqNo is assigned at send time** (writer, auto mode). Buffered messages carry `seqNo=0n` and get numbered in `pump`'s `formBatch` — so a reconnect never renumbers, - and the `closing` pump gate (`hasEverConnected`) guarantees no auto seqNo is ever - assigned before the server's high-water mark is recovered. + and only `ready` can send, including during a graceful close. No auto seqNo is assigned before the server watermark is recovered. Pinned by: writer-state tests, `writer-protocol.test.ts` (live dedup proof). 2. **commit() is never rejected by a transparent reconnect** (reader). Pending commits are buffered per partition (keyed by `partitionKey` — path + partitionId, so a multi-topic reader's equal partition ids never collide) and their exact remaining ranges are re-sent after the start handshake on the new session. A commit racing a stopped partition rejects immediately; pending commits can also reject on the `partition_reassign_gc` timer or terminal shutdown. Pinned by: `reader.model.test.ts` (800-seed invariant), `reader.multi-topic.test.ts`. -3. **Grant epoch** (`grantId` + `ackPending` / `stopReady`): a partition grant is live +3. **Grant identity** (session object + `ackPending` / `stopReady`): a partition grant is live only after `start_ready` → `start_response`, and a graceful stop is answered only after `stop_ready` + drained commits; buffered commits are sent exactly once, and stale hook completions (session ids restart at 1 per stream) can never double-answer. @@ -383,12 +275,12 @@ stateDiagram-v2 4. **Commit ranges are stitched at delivery time** (`deliveredWatermark` + per-message `commitRangeStart`): a server-side offset hole (retention, readFrom skip) is attributed to the next delivered message, and a delivered-but-unacked - message is never covered by another message's commit. `claimedRanges` subtracts + message is never covered by another message's commit. `pendingRanges` subtracts already-sent coverage, so a zero-width, inverted, or overlapping commit range (session-fatal server-side) is unrepresentable. Pinned by: `reader.commit-semantics.test.ts`, stitching tests in `reader-state.test.ts`, model invariant. -5. **Flow control is charged per ReadResponse** and released only after all messages from that response have passed through `read()` or been discarded for a stopped session; credit is re-granted in batches of ≥ maxBufferBytes/5. +5. **Flow control uses whole server response sizes**: each `ReadResponse.bytesSize` is charged and released exactly once, after its final message passes through `read()` or is discarded for a stopped session. Initialized streams refund that exact size, never a sum of payload sizes. Reconnect keeps `bufferedBytes`; `connecting` enters `ready` only after InitResponse and the first positive `maxBufferBytes - bufferedBytes` grant. Before InitResponse and during backoff, releases send nothing. 6. **Terminal errors throw from the facade, never end streams silently**: the output queue drains buffered facts first, then throws (`AsyncQueue.fail`), and every later `read()`/`commit()`/`close()` rethrows the same reason. diff --git a/packages/topic/README.md b/packages/topic/README.md index 380a501b..ddc01827 100644 --- a/packages/topic/README.md +++ b/packages/topic/README.md @@ -90,7 +90,7 @@ await using writer = createTopicWriter(driver, { - `topic`: `string | TopicReaderSource | TopicReaderSource[]` — topic path or detailed sources - `consumer`: `string` — consumer name - `codecMap?`: `Map` — custom codecs for decompression (built-ins: RAW, GZIP, ZSTD) -- `maxBufferBytes?`: `bigint` — initial server read-credit window (default 8 MiB); an oversized message may exceed it +- `maxBufferBytes?`: `bigint` — read-credit budget shared with buffered responses across reconnects (default 8 MiB); an oversized message may exceed it - `updateTokenIntervalMs?`: `number` — auth token refresh interval (default 60000) - `gracefulShutdownTimeoutMs?`: `number` — force-close deadline for graceful `close()` before pending commits are dropped (default 30000) - `recoveryWindowMs?`: `number` — terminal reconnect window; unbounded by default (reconnect forever, waiting for the server/topic), pass a finite ms value to bound it @@ -102,6 +102,8 @@ await using writer = createTopicWriter(driver, { `reader.bufferedBytes` exposes the server-accounted bytes currently retained by the reader and not yet fully delivered through `read()`. +Buffered messages remain encoded until `read()` selects them for delivery. `maxBufferBytes` limits server read credit, not decompressed memory or RSS; use `read({ limit })` to bound the number of messages decoded for one batch. A clean `close()` preserves the unread tail, while `destroy()` and terminal errors discard it. Decoding errors surface when the affected message is read. + TopicReaderSource supports partition filters and time‑based selectors: ```ts @@ -146,9 +148,9 @@ Commit semantics: each message acknowledges its own offset range (plus any serve - `tx?`: `TX` — transaction to write within - `producer?`: `string` — producer id (auto‑generated if omitted) - `codec?`: `CompressionCodec` — compression (default RAW; built-ins: RAW, GZIP, ZSTD) -- `maxBufferBytes?`: `bigint` — writer buffer cap (default 256 MiB) +- `maxBufferBytes?`: `bigint` — limit for unacknowledged payload bytes after compression (default 256 MiB); metadata and JavaScript object overhead are not included - `maxInflightCount?`: `number` — max messages in‑flight (default 1000) -- `flushIntervalMs?`: `number` — periodic flush tick (default 1000ms) +- `flushIntervalMs?`: `number` — periodic flush of a partially filled batch (default 1000ms) - `updateTokenIntervalMs?`: `number` — auth token refresh interval (default 60000) - `gracefulShutdownTimeoutMs?`: `number` — force-close deadline for graceful `close()` (default 30000) - `recoveryWindowMs?`: `number` — terminal reconnect window; unbounded by default (reconnect forever, waiting for the server/topic), pass a finite ms value to bound it @@ -178,7 +180,11 @@ writer.write(payload) // fire-and-forget (void) const lastSeqNo = await writer.flush() ``` -`write()` accepts `Uint8Array` only. Encode your own objects/strings as needed. +`write()` accepts `Uint8Array` only. Encode your own objects/strings as needed. With RAW, the writer may retain your buffer without copying it. The same unchanged buffer may be passed to multiple `write()` calls, including while earlier writes are awaiting acknowledgment. Do not change its bytes or detach its backing buffer until all writes using it are acknowledged through `onAck` or a completed `flush()` called after those writes. Metadata and timestamps are copied when `write()` accepts the message. + +`flush()` waits for acknowledgments of the writes accepted before that call and returns the sequence number at that boundary. Later `write()` calls do not extend its wait. Concurrent flush calls may cover different boundaries; a terminal failure rejects any that are still pending. `close()` stops accepting writes and drains the entire queue. + +Acknowledgment observers (`onAck` and the reader's `onCommittedOffset`) do not delay flush or commit completion. Synchronous exceptions and rejected async callbacks are logged; use an explicit application task when callback work must finish before shutdown. ## Transactions @@ -245,7 +251,7 @@ await using writer = createTopicWriter(driver, { - `@ydbjs/topic`: `topic(driver)` and types - `@ydbjs/topic/reader`: `createTopicReader`, `createTopicTxReader`, reader types -- `@ydbjs/topic/writer`: `createTopicWriter`, `createTopicTxWriter`, writer types +- `@ydbjs/topic/writer`: `createTopicWriter`, `createTopicTxWriter`, writer types including `TopicTxWriter` ## License diff --git a/packages/topic/README.ru.md b/packages/topic/README.ru.md index 235a942d..e1826571 100644 --- a/packages/topic/README.ru.md +++ b/packages/topic/README.ru.md @@ -88,7 +88,7 @@ await using writer = createTopicWriter(driver, { - `topic`: `string | TopicReaderSource | TopicReaderSource[]` — путь или источники с фильтрами - `consumer`: `string` — имя консюмера - `codecMap?`: `Map` — свои кодеки для распаковки (встроенные: RAW, GZIP, ZSTD) -- `maxBufferBytes?`: `bigint` — начальное окно чтения на сервере (по умолчанию 8 МиБ); большое сообщение может его превысить +- `maxBufferBytes?`: `bigint` — бюджет чтения с учётом буферизованных ответов, сохраняемый при переподключении (по умолчанию 8 МиБ); большое сообщение может его превысить - `updateTokenIntervalMs?`: `number` — период обновления токена (по умолчанию 60000) - `gracefulShutdownTimeoutMs?`: `number` — дедлайн принудительного закрытия для graceful `close()`, после него ожидающие коммиты отбрасываются (по умолчанию 30000) - `recoveryWindowMs?`: `number` — окно реконнекта; по умолчанию неограниченно (реконнект вечно, ждём сервер/топик), передайте конечное значение в мс, чтобы ограничить @@ -100,6 +100,8 @@ await using writer = createTopicWriter(driver, { `reader.bufferedBytes` показывает рассчитанный сервером объём данных, который ридер сейчас удерживает и ещё не полностью выдал через `read()`. +Сообщения остаются в закодированном виде до выдачи через `read()`. `maxBufferBytes` ограничивает кредит чтения, а не объём распакованных данных или RSS; `read({ limit })` ограничивает число сообщений, декодируемых для одного батча. Обычный `close()` сохраняет непрочитанный хвост, а `destroy()` и терминальные ошибки удаляют его. Ошибка декодирования возникает при чтении соответствующего сообщения. + TopicReaderSource поддерживает фильтры партиций и временные селекторы: ```ts @@ -142,9 +144,9 @@ for await (const batch of reader.read({ limit: 100, batchWindowMs: 1000 })) { - `tx?`: `TX` — транзакция для записи - `producer?`: `string` — id продюсера (по умолчанию генерируется) - `codec?`: `CompressionCodec` — сжатие (RAW/GZIP/ZSTD или своё) -- `maxBufferBytes?`: `bigint` — лимит буфера (по умолчанию 256 МБ) +- `maxBufferBytes?`: `bigint` — лимит неподтверждённых payload после сжатия (по умолчанию 256 МиБ); metadata и служебные JS-объекты в него не входят - `maxInflightCount?`: `number` — максимум сообщений «в полёте» (по умолчанию 1000) -- `flushIntervalMs?`: `number` — периодический флаш (по умолчанию 1000 мс) +- `flushIntervalMs?`: `number` — периодическая отправка неполного батча (по умолчанию 1000 мс) - `updateTokenIntervalMs?`: `number` — период обновления токена (по умолчанию 60000) - `gracefulShutdownTimeoutMs?`: `number` — дедлайн принудительного закрытия для graceful `close()` (по умолчанию 30000) - `recoveryWindowMs?`: `number` — окно реконнекта; по умолчанию неограниченно (реконнект вечно, ждём сервер/топик), передайте конечное значение в мс, чтобы ограничить @@ -234,7 +236,7 @@ await using writer = createTopicWriter(driver, { - `@ydbjs/topic`: `topic(driver)` и типы - `@ydbjs/topic/reader`: `createTopicReader`, `createTopicTxReader` и типы -- `@ydbjs/topic/writer`: `createTopicWriter`, `createTopicTxWriter` и типы +- `@ydbjs/topic/writer`: `createTopicWriter`, `createTopicTxWriter` и типы, включая `TopicTxWriter` ## Лицензия diff --git a/packages/topic/src/memory-profile-verifier.test.ts b/packages/topic/src/memory-profile-verifier.test.ts new file mode 100644 index 00000000..dcc348b6 --- /dev/null +++ b/packages/topic/src/memory-profile-verifier.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from 'vitest' + +import { verifyMemoryProfile } from '../tests/memory-profile.ts' +import type { MemoryReport } from '../tests/memory-worker.ts' + +let scenarios = ['steady', 'reconnect', 'replace'] +let runtimes = ['node', 'bun'] as const + +function memoryReport(runtime: 'node' | 'bun'): MemoryReport { + return { + runtime: { name: runtime, version: 'fixture', supportsActiveResources: runtime === 'node' }, + node: 'fixture', + platform: 'fixture', + arch: 'fixture', + epochs: 9, + epochPauseMs: 0, + payloadBytes: 32 * 1024, + messagesPerEpoch: 128, + accepted: [1], + acknowledged: [1], + delivered: [1], + committed: [1], + reconnects: { reader: 1, writer: 1 }, + closedClientsAlive: 0, + retainedControlBytes: 0, + samples: scenarios.flatMap((scenario) => + Array.from({ length: 9 }, (_, epoch) => ({ + phase: 'drained', + scenario, + epoch, + rss: 128 * 1024 * 1024, + heapUsed: 16 * 1024 * 1024, + heapTotal: 32 * 1024 * 1024, + external: 4 * 1024 * 1024, + arrayBuffers: 4 * 1024 * 1024, + activeResources: [], + jsc: { + heapSize: 16 * 1024 * 1024, + extraMemorySize: 4 * 1024 * 1024, + objectCount: 100, + }, + })) + ), + } +} + +test.for(runtimes.flatMap((runtime) => scenarios.map((scenario) => ({ runtime, scenario }))))( + 'rejects RSS growth with stable heap and buffers in $runtime/$scenario', + ({ runtime, scenario }) => { + let report = memoryReport(runtime) + expect(() => verifyMemoryProfile(report, 1)).not.toThrow() + + for (let sample of report.samples) { + if (sample.scenario === scenario && sample.epoch >= 6) { + sample.rss += 1024 * 1024 * 1024 + } + } + + expect(() => verifyMemoryProfile(report, 1)).toThrow(`${scenario} rss grew beyond 64 MiB`) + } +) + +test.for(runtimes)('allows bounded RSS growth after allocator warmup on %s', (runtime) => { + let report = memoryReport(runtime) + + for (let sample of report.samples) { + if (sample.epoch < 3) { + sample.rss += 1024 * 1024 * 1024 + } else if (sample.epoch >= 6) { + sample.rss += 32 * 1024 * 1024 + } + } + + report.samples.push({ ...report.samples.at(-1)!, phase: 'loaded', rss: 2 * 1024 * 1024 * 1024 }) + + expect(() => verifyMemoryProfile(report, 1)).not.toThrow() +}) diff --git a/packages/topic/src/memory-profile.test.ts b/packages/topic/src/memory-profile.test.ts new file mode 100644 index 00000000..0d0e56e0 --- /dev/null +++ b/packages/topic/src/memory-profile.test.ts @@ -0,0 +1,191 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' + +import { Server, ServerCredentials, type ServerUnaryCall, type sendUnaryData } from '@grpc/grpc-js' +import { expect, test } from 'vitest' + +import { runMemoryProfile } from '../tests/memory-profile.ts' + +let execute = promisify(execFile) +let grpcPath = createRequire(import.meta.url).resolve('@grpc/grpc-js') +let method = { + path: '/memory.Proxy/Probe', + requestStream: false, + responseStream: false, + requestSerialize: (value: string) => Buffer.from(value), + requestDeserialize: (value: Buffer) => value.toString(), + responseSerialize: (value: string) => Buffer.from(value), + responseDeserialize: (value: Buffer) => value.toString(), +} + +// Certificate failures need a local server with a deliberately invalid identity. +async function proxyFixture(signal: AbortSignal, certificateName?: string, trusted = true) { + let directory = await mkdtemp(join(tmpdir(), 'topic-memory-proxy-')) + let server = new Server() + let environment = new Map( + ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH', 'no_grpc_proxy', 'no_proxy'].map((key) => [ + key, + process.env[key], + ]) + ) + let dispose = async () => { + server.forceShutdown() + for (let [key, value] of environment) { + if (value === undefined) { + delete process.env[key] + } else { + process.env[key] = value + } + } + await rm(directory, { recursive: true, force: true }) + } + try { + let credentials = ServerCredentials.createInsecure() + delete process.env['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] + if (certificateName) { + let key = join(directory, 'key.pem') + let cert = join(directory, 'cert.pem') + await execute( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + key, + '-out', + cert, + '-subj', + `/CN=${certificateName}`, + '-addext', + `subjectAltName=DNS:${certificateName}`, + '-days', + '1', + ], + { signal } + ) + credentials = ServerCredentials.createSsl(null, [ + { private_key: await readFile(key), cert_chain: await readFile(cert) }, + ]) + if (trusted) { + process.env['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert + } + } + process.env['no_grpc_proxy'] = 'localhost' + process.env['no_proxy'] = 'localhost' + let calls = 0 + server.addService( + { probe: method }, + { + probe(call: ServerUnaryCall, callback: sendUnaryData) { + calls++ + callback(null, call.getPeer()) + }, + } + ) + let port = await new Promise((resolve, reject) => { + server.bindAsync('localhost:0', credentials, (error, boundPort) => { + if (error) { + reject(error) + } else { + resolve(boundPort) + } + }) + }) + let worker = join(directory, 'worker.cjs') + await writeFile( + worker, + `let assert = require('node:assert/strict') +let { once } = require('node:events') +let { writeFileSync } = require('node:fs') +let grpc = require(${JSON.stringify(grpcPath)}) +process.channel.ref() +let url = new URL(process.env.YDB_CONNECTION_STRING) +let Client = grpc.makeGenericClientConstructor({ probe: { + path: '/memory.Proxy/Probe', requestStream: false, responseStream: false, + requestSerialize: value => Buffer.from(value), responseDeserialize: value => value.toString(), +} }) +let client = new Client(url.host, url.protocol === 'grpcs:' ? grpc.credentials.createSsl() : grpc.credentials.createInsecure()) +let probe = () => new Promise((resolve, reject) => client.probe('ping', (error, reply) => error ? reject(error) : resolve(reply))) +async function run() { + try { + let first = await probe() + let disconnected = once(process, 'message') + process.send({ type: 'disconnect' }) + await disconnected + let channel = client.getChannel() + if (channel.getConnectivityState(false) === grpc.connectivityState.READY) { + await new Promise((resolve, reject) => channel.watchConnectivityState(grpc.connectivityState.READY, Infinity, error => error ? reject(error) : resolve())) + } + await new Promise((resolve, reject) => client.waitForReady(Infinity, error => error ? reject(error) : resolve())) + let second = await probe() + assert.notEqual(first, second, 'Disconnect did not replace the upstream TCP connection') + writeFileSync(process.env.MEMORY_REPORT_FILE, JSON.stringify({ node: process.version })) + } finally { + client.close() + process.disconnect() + } +} +run().catch(error => { console.error(error); process.exitCode = 1 }) +` + ) + return { + connectionString: `${certificateName ? 'grpcs' : 'grpc'}://localhost:${port}/local`, + worker, + report: join(directory, 'report.json'), + get calls() { + return calls + }, + [Symbol.asyncDispose]: dispose, + } + } catch (error) { + await dispose() + throw error + } +} + +test('cuts plaintext memory worker connections despite inherited proxy exclusions', async (tc) => { + await using fixture = await proxyFixture(tc.signal) + let result = await runMemoryProfile( + fixture.connectionString, + fixture.worker, + fixture.report, + tc.signal + ) + expect(result.cuts).toBe(1) + expect(fixture.calls).toBe(2) +}) + +test('preserves TLS server identity when cutting memory worker connections', async (tc) => { + await using fixture = await proxyFixture(tc.signal, 'localhost') + let result = await runMemoryProfile( + fixture.connectionString, + fixture.worker, + fixture.report, + tc.signal + ) + expect(result.cuts).toBe(1) + expect(fixture.calls).toBe(2) +}) + +test('rejects a trusted certificate for the wrong memory upstream hostname', async (tc) => { + await using fixture = await proxyFixture(tc.signal, 'wrong.example') + await expect( + runMemoryProfile(fixture.connectionString, fixture.worker, fixture.report, tc.signal) + ).rejects.toThrow(/Hostname\/IP does not match certificate/) + expect(fixture.calls).toBe(0) +}) + +test('rejects an untrusted memory upstream certificate', async (tc) => { + await using fixture = await proxyFixture(tc.signal, 'localhost', false) + await expect( + runMemoryProfile(fixture.connectionString, fixture.worker, fixture.report, tc.signal) + ).rejects.toThrow(/self-signed certificate/) + expect(fixture.calls).toBe(0) +}) diff --git a/packages/topic/src/partition-session.ts b/packages/topic/src/partition-session.ts index fc4dd9af..8e588c8f 100644 --- a/packages/topic/src/partition-session.ts +++ b/packages/topic/src/partition-session.ts @@ -16,7 +16,7 @@ export class TopicPartitionSession { */ partitionOffsets = { start: 0n, end: 0n } /** - * Offset of the last committed message from the partition. + * Exclusive committed offset: messages below it are committed. */ partitionCommittedOffset = 0n /** diff --git a/packages/topic/src/reader/diagnostics.ts b/packages/topic/src/reader/diagnostics.ts index d2b694ed..da0f18f5 100644 --- a/packages/topic/src/reader/diagnostics.ts +++ b/packages/topic/src/reader/diagnostics.ts @@ -15,8 +15,6 @@ export type ReaderScope = { topics: string[] } -// One-shot config snapshot, published on `opened` so late-joining subscribers still -// learn the reader's effective configuration. export type ReaderConfig = { maxBufferBytes: bigint updateTokenIntervalMs: number diff --git a/packages/topic/src/reader/read-settings.test.ts b/packages/topic/src/reader/read-settings.test.ts index db5cf960..7463b1e5 100644 --- a/packages/topic/src/reader/read-settings.test.ts +++ b/packages/topic/src/reader/read-settings.test.ts @@ -36,10 +36,10 @@ test('parses an ms-string maxLag', () => { expect(settings!.maxLag).toMatchObject({ seconds: 90n, nanos: 0 }) }) -test('passes a Duration maxLag through untouched', () => { +test('preserves the value of a Duration maxLag', () => { let duration: Duration = create(DurationSchema, { seconds: 5n, nanos: 250 }) let [settings] = parseReadSettings({ path: '/t', maxLag: duration }) - expect(settings!.maxLag).toBe(duration) + expect(settings!.maxLag).toEqual(duration) }) test('sends an explicit zero maxLag on the wire', () => { diff --git a/packages/topic/src/reader/read-settings.ts b/packages/topic/src/reader/read-settings.ts index 88ff4089..6a74fcad 100644 --- a/packages/topic/src/reader/read-settings.ts +++ b/packages/topic/src/reader/read-settings.ts @@ -1,4 +1,4 @@ -import { create, protoInt64 } from '@bufbuild/protobuf' +import { clone, create, protoInt64 } from '@bufbuild/protobuf' import { type Duration, DurationSchema, @@ -80,5 +80,7 @@ export let parseReadSettings = function parseReadSettings( } } - return settings + return settings.map((setting) => + clone(StreamReadMessage_InitRequest_TopicReadSettingsSchema, setting) + ) } diff --git a/packages/topic/src/reader/reader-runtime.ts b/packages/topic/src/reader/reader-runtime.ts index 6a3e2921..48dcf413 100644 --- a/packages/topic/src/reader/reader-runtime.ts +++ b/packages/topic/src/reader/reader-runtime.ts @@ -6,8 +6,7 @@ import { loggers } from '@ydbjs/debug' import { type MachineRuntime, createMachineRuntime } from '@ydbjs/fsm' import { parseReadSettings } from './read-settings.js' -import { PRIORITY_CONTROL, ReaderTransport } from './transport.js' -import type { TransportOutput } from './transport-state.js' +import { PRIORITY_CONTROL, ReaderTransport, type TransportOutput } from './transport.js' import { type ReaderCtx, type ReaderEffect, @@ -18,6 +17,7 @@ import { type TimerRef, createReaderCtx, readerTransition, + releaseState, } from './reader-state.js' import type { TopicReaderOptions, @@ -47,14 +47,8 @@ type ReaderEnv = { onPartitionSessionStart?: onPartitionSessionStartCallback onPartitionSessionStop?: onPartitionSessionStopCallback - startTimeoutMs: number updateTokenIntervalMs: number gracefulShutdownTimeoutMs: number - partitionReassignGcMs: number - - ac: AbortController - closedDeferred: PromiseWithResolvers - isFinalized: boolean timers: Map> } @@ -100,7 +94,7 @@ let backoffDelay = function backoffDelay(attempts: number): number { let delayFor = function delayFor(ctx: FullCtx, which: TimerName): number { switch (which) { case 'start_timeout': - return ctx.startTimeoutMs + return DEFAULT_START_TIMEOUT_MS case 'retry_backoff': return backoffDelay(ctx.attempts) case 'recovery_window': @@ -111,7 +105,7 @@ let delayFor = function delayFor(ctx: FullCtx, which: TimerName): number { case 'partition_graceful_timeout': return ctx.gracefulShutdownTimeoutMs case 'partition_reassign_gc': - return ctx.partitionReassignGcMs + return DEFAULT_PARTITION_REASSIGN_GC_MS case 'partition_commit_status': return COMMIT_STATUS_INTERVAL_MS } @@ -131,6 +125,17 @@ let clearTimerByKey = function clearTimerByKey(ctx: FullCtx, key: string): void } } +let finalize = function finalize(ctx: FullCtx, reason: unknown): void { + for (let handle of ctx.timers.values()) { + clearTimeout(handle) + } + ctx.timers.clear() + releaseState(ctx) + delete ctx.onPartitionSessionStart + delete ctx.onPartitionSessionStop + ctx.transport.destroy(reason) +} + let mapTransportOutput = function mapTransportOutput(output: TransportOutput): ReaderEvent | null { evdbg.log('transport → reader: %s', output.type) switch (output.type) { @@ -225,15 +230,9 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) onPartitionSessionStop: options.onPartitionSessionStop, }), - startTimeoutMs: DEFAULT_START_TIMEOUT_MS, updateTokenIntervalMs: options.updateTokenIntervalMs ?? DEFAULT_UPDATE_TOKEN_INTERVAL_MS, gracefulShutdownTimeoutMs: options.gracefulShutdownTimeoutMs ?? DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_MS, - partitionReassignGcMs: DEFAULT_PARTITION_REASSIGN_GC_MS, - - ac: new AbortController(), - closedDeferred: Promise.withResolvers(), - isFinalized: false, timers: new Map(), } @@ -329,10 +328,6 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) }) }, - 'reader.effect.transport.close': (fullCtx) => { - fullCtx.transport.close() - }, - // The one effect where the runtime makes a domain decision, by necessity: the // StartPartitionSessionResponse's read_offset/commit_offset come from the async // `onPartitionSessionStart` hook (typically an external offset-store lookup — I/O), @@ -346,11 +341,11 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) // result re-enters the machine as reader.partition.start_ready, so the // answer-or-not decision is made against fresh state in the transition. void (async () => { - let session = fullCtx.partitions.get(effect.partitionKey)?.session + let session = effect.session let readOffset: bigint | undefined let commitOffset: bigint | undefined - if (fullCtx.onPartitionSessionStart && session) { + if (fullCtx.onPartitionSessionStart) { try { let result = await fullCtx.onPartitionSessionStart( session, @@ -368,9 +363,7 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) runtime.dispatch({ type: 'reader.partition.start_ready', - partitionSessionId: effect.partitionSessionId, - partitionKey: effect.partitionKey, - grantId: effect.grantId, + session, ...(readOffset !== undefined && { readOffset }), ...(commitOffset !== undefined && { commitOffset }), }) @@ -383,9 +376,9 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) // response goes out only after stop_ready AND the pending commits drain. 'reader.effect.partition.stop_hook': (fullCtx, effect, runtime) => { void (async () => { - let session = fullCtx.partitions.get(effect.partitionKey)?.session + let session = effect.session - if (fullCtx.onPartitionSessionStop && session) { + if (fullCtx.onPartitionSessionStop) { try { await fullCtx.onPartitionSessionStop(session, effect.committedOffset) } catch (error) { @@ -395,8 +388,7 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) runtime.dispatch({ type: 'reader.partition.stop_ready', - partitionKey: effect.partitionKey, - grantId: effect.grantId, + session, }) })() }, @@ -451,23 +443,14 @@ export function createReaderRuntime(driver: Driver, options: TopicReaderOptions) clearTimerByKey(fullCtx, timerKey(effect)) }, - 'reader.effect.finalize': (fullCtx, effect) => { - if (fullCtx.isFinalized) { - return - } - fullCtx.isFinalized = true - - for (let handle of fullCtx.timers.values()) { - clearTimeout(handle) - } - fullCtx.timers.clear() - - fullCtx.transport.destroy(effect.reason) - fullCtx.ac.abort(effect.reason) - fullCtx.closedDeferred.resolve() - }, + 'reader.effect.finalize': (fullCtx, effect) => finalize(fullCtx, effect.reason), }, }) + machine.signal.addEventListener( + 'abort', + () => finalize(ctx as FullCtx, machine.signal.reason), + { once: true } + ) machine.ingest(transport.events, mapTransportOutput) diff --git a/packages/topic/src/reader/reader-state.test.ts b/packages/topic/src/reader/reader-state.test.ts index 6dff931e..1916c9e8 100644 --- a/packages/topic/src/reader/reader-state.test.ts +++ b/packages/topic/src/reader/reader-state.test.ts @@ -1,7 +1,9 @@ import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { YDBError } from '@ydbjs/error' import { expect, test } from 'vitest' +import { ClientError, Status } from 'nice-grpc' +import { isRetryableTopicError } from '../retry.ts' import { type OffsetRange, type ReaderCtx, @@ -10,7 +12,6 @@ import { type ReaderOutput, type ReaderState, createReaderCtx, - isRetryableReaderError, partitionKey, readerTransition, } from './reader-state.ts' @@ -175,11 +176,10 @@ let commit = function commit( // Complete the async start handshake for the partition's CURRENT grant — until the // ack, commits buffer (ackPending) instead of hitting the wire. let ackStart = function ackStart(h: Harness, sessionId: bigint, partitionId: bigint): void { + expect(h.ctx.partitions.get(pk(partitionId))!.session.partitionSessionId).toBe(sessionId) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: sessionId, - partitionKey: pk(partitionId), - grantId: h.ctx.partitions.get(pk(partitionId))!.grantId, + session: h.ctx.partitions.get(pk(partitionId))!.session, }) } @@ -187,8 +187,7 @@ let ackStart = function ackStart(h: Harness, sessionId: bigint, partitionId: big let ackStop = function ackStop(h: Harness, partitionId: bigint): void { step(h, { type: 'reader.partition.stop_ready', - partitionKey: pk(partitionId), - grantId: h.ctx.partitions.get(pk(partitionId))!.grantId, + session: h.ctx.partitions.get(pk(partitionId))!.session, }) } @@ -288,7 +287,7 @@ test('errors terminally on a fatal disconnect', () => { }) test('classifies SCHEME_ERROR as retryable when retryOnSchemeError is set', () => { - expect(isRetryableReaderError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []), true)).toBe( + expect(isRetryableTopicError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []), true)).toBe( true ) }) @@ -341,9 +340,7 @@ test('keys same-numbered partitions from different topics separately', () => { expect(h.ctx.sessionIndex.get(2n)).toBe(partitionKey('/b', 1n)) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: partitionKey('/a', 1n), - grantId: h.ctx.partitions.get(partitionKey('/a', 1n))!.grantId, + session: h.ctx.partitions.get(partitionKey('/a', 1n))!.session, }) step(h, { type: 'reader.commit', @@ -371,7 +368,7 @@ test('delivers messages and charges the flow-control budget', () => { expect(delivered).toHaveLength(1) expect(delivered[0]!.groups[0]!.messages.map((m) => m.offset)).toEqual([5n, 6n, 7n]) expect(delivered[0]!.releaseBytes).toBe(300n) - expect(h.ctx.inFlightBytes).toBe(300n) + expect(h.ctx.bufferedBytes).toBe(300n) }) test('stitches the head gap into the first delivered message commit range', () => { @@ -484,7 +481,7 @@ test('charges a multi-partition read response only once', () => { expect(delivered).toHaveLength(1) expect(delivered[0]!.groups).toHaveLength(2) expect(delivered[0]!.releaseBytes).toBe(1000n) // once, not 1000 per partition - expect(h.ctx.inFlightBytes).toBe(1000n) + expect(h.ctx.bufferedBytes).toBe(1000n) }) test('drops the superseded session id from the index on reassign', () => { @@ -541,10 +538,8 @@ test('sends the commit ranges verbatim and records the pending commit', () => { expect(cs).toHaveLength(1) expect(cs[0]!.ranges).toEqual([{ start: 5n, end: 9n }]) let entry = h.ctx.partitions.get(pk(10n))! - expect(entry.pendingCommits).toEqual([ - { targetOffset: 9n, wireRanges: [{ start: 5n, end: 9n }], waiterId: 1 }, - ]) - expect(entry.claimedRanges).toEqual([{ start: 5n, end: 9n }]) + expect(entry.commitWaiters).toEqual([{ targetOffset: 9n, waiterId: 1 }]) + expect(entry.pendingRanges).toEqual([{ start: 5n, end: 9n }]) }) test('clamps a commit range below the server committed offset', () => { @@ -568,7 +563,7 @@ test('resolves without a wire send when the range is fully below the committed o commit(h, 10n, [{ start: 0n, end: 5n }], 7) expect(commitSends(h.effects)).toHaveLength(0) expect(outputs(h, 'reader.commit.resolved').map((o) => o.waiterId)).toEqual([7]) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(0) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(0) }) test('waits for the server watermark when the range is fully claimed by an earlier commit', () => { @@ -582,9 +577,9 @@ test('waits for the server watermark when the range is fully claimed by an earli commit(h, 10n, [{ start: 5n, end: 7n }], 2) expect(commitSends(h.effects)).toHaveLength(0) expect(outputs(h, 'reader.commit.resolved')).toHaveLength(0) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toEqual([ - { targetOffset: 7n, wireRanges: [{ start: 5n, end: 7n }], waiterId: 1 }, - { targetOffset: 7n, wireRanges: [], waiterId: 2 }, + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toEqual([ + { targetOffset: 7n, waiterId: 1 }, + { targetOffset: 7n, waiterId: 2 }, ]) h.emitted.length = 0 @@ -602,12 +597,11 @@ test('subtracts claimed coverage and sends only the remainder', () => { expect(cs).toHaveLength(1) expect(cs[0]!.ranges).toEqual([{ start: 6n, end: 8n }]) let entry = h.ctx.partitions.get(pk(10n))! - expect(entry.pendingCommits[1]).toEqual({ + expect(entry.commitWaiters[1]).toEqual({ targetOffset: 8n, - wireRanges: [{ start: 6n, end: 8n }], waiterId: 2, }) - expect(entry.claimedRanges).toEqual([{ start: 5n, end: 8n }]) + expect(entry.pendingRanges).toEqual([{ start: 5n, end: 8n }]) }) test('merges overlapping and adjacent input ranges before the send', () => { @@ -639,7 +633,7 @@ test('resolves a commit on a covering commit-offset response', () => { h.emitted.length = 0 message(h, commitMsg([[1n, 9n]])) expect(outputs(h, 'reader.commit.resolved')[0]).toMatchObject({ waiterId: 1 }) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(0) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(0) }) test('keeps a commit pending until the high-water mark reaches its end', () => { @@ -650,9 +644,9 @@ test('keeps a commit pending until the high-water mark reaches its end', () => { message(h, commitMsg([[1n, 7n]])) // below end 9 expect(outputs(h, 'reader.commit.resolved')).toHaveLength(0) let entry = h.ctx.partitions.get(pk(10n))! - expect(entry.pendingCommits).toHaveLength(1) + expect(entry.commitWaiters).toHaveLength(1) // server-confirmed coverage is compacted away so the claim list stays bounded - expect(entry.claimedRanges).toEqual([{ start: 7n, end: 9n }]) + expect(entry.pendingRanges).toEqual([{ start: 7n, end: 9n }]) }) test('emits partition.committed with the session on a commit ack', () => { @@ -669,6 +663,37 @@ test('emits partition.committed with the session on a commit ack', () => { // ── reconnect reconcile (THE CRUX) ──────────────────────────────────────────────── +test('replays partition ranges once while commit callers retain separate watermarks', () => { + let h = mk() + toReadyWithPartition(h) + ackStart(h, 1n, 10n) + commit(h, 10n, [{ start: 5n, end: 7n }], 1) + commit(h, 10n, [{ start: 5n, end: 9n }], 2) + commit(h, 10n, [{ start: 12n, end: 13n }], 3) + step(h, { type: 'reader.stream.disconnected' }) + step(h, { type: 'reader.timer.retry_backoff' }) + step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) + message(h, startMsg(2n, 10n, 6n)) + ackStart(h, 2n, 10n) + expect(commitSends(h.effects)).toEqual([ + { + type: 'reader.effect.send.commit', + partitionSessionId: 2n, + ranges: [ + { start: 6n, end: 9n }, + { start: 12n, end: 13n }, + ], + }, + ]) + h.emitted.length = 0 + message(h, commitMsg([[2n, 7n]])) + expect(outputs(h, 'reader.commit.resolved').map((output) => output.waiterId)).toEqual([1]) + expect(h.ctx.partitions.get(pk(10n))!.pendingRanges).toEqual([ + { start: 7n, end: 9n }, + { start: 12n, end: 13n }, + ]) +}) + test('re-sends an unacked commit on the new session after reconnect', () => { let h = mk() toReadyWithPartition(h) // session 1, committed 5 @@ -697,11 +722,11 @@ test('re-sends an unacked commit on the new session after reconnect', () => { expect(cs).toHaveLength(1) expect(cs[0]!.partitionSessionId).toBe(2n) expect(cs[0]!.ranges).toEqual([{ start: 5n, end: 9n }]) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(1) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(1) expect(outputs(h, 'reader.commit.resolved')).toHaveLength(0) }) -test('replays each pending commit with its exact remaining ranges on resend', () => { +test('replays sparse partition ranges without covering unrequested offsets', () => { let h = mk() toReadyWithPartition(h) ackStart(h, 1n, 10n) @@ -718,7 +743,12 @@ test('replays each pending commit with its exact remaining ranges on resend', () // One send per pending, ranges verbatim — a gap-covering [5,9) span would commit // the delivered-but-unacked offsets 6 and 7 behind the app's back. let cs = commitSends(h.effects) - expect(cs.map((e) => e.ranges)).toEqual([[{ start: 5n, end: 6n }], [{ start: 8n, end: 9n }]]) + expect(cs.map((e) => e.ranges)).toEqual([ + [ + { start: 5n, end: 6n }, + { start: 8n, end: 9n }, + ], + ]) }) test('sends a commit under the new session id after a within-stream regrant', () => { @@ -776,7 +806,7 @@ test('blocks a commit when the reused session id belongs to another partition', // Partition 10's entry still holds sessionId 1, but that id now belongs to // partition 20 — must buffer, never send under the colliding id. expect(commitSends(h.effects)).toHaveLength(0) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(1) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(1) }) test('resolves a pending commit that the server committed before the reconnect', () => { @@ -793,7 +823,7 @@ test('resolves a pending commit that the server committed before the reconnect', // server now reports committed 9 (the commit made it durable before the drop) message(h, startMsg(2n, 10n, 9n)) expect(outputs(h, 'reader.commit.resolved')[0]).toMatchObject({ waiterId: 1 }) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(0) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(0) }) test('never rejects a pending commit merely because the stream reconnected', () => { @@ -818,7 +848,7 @@ test('holds pending commits on a non-graceful stop and schedules a gc timer', () h.emitted.length = 0 message(h, stopMsg(1n, false)) expect(h.ctx.partitions.get(pk(10n))!.state).toBe('stopped') - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(1) // held, not rejected + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(1) // held, not rejected expect(outputs(h, 'reader.commit.rejected')).toHaveLength(0) expect(h.effects).toContainEqual({ type: 'reader.effect.timer.schedule', @@ -830,7 +860,7 @@ test('holds pending commits on a non-graceful stop and schedules a gc timer', () test('enters stopping-graceful and runs the stop hook even with no pending commits', () => { let h = mk() toReadyWithPartition(h) - let grantId = h.ctx.partitions.get(pk(10n))!.grantId + let session = h.ctx.partitions.get(pk(10n))!.session h.emitted.length = 0 message(h, stopMsg(1n, true)) // The stop response waits for the async onPartitionSessionStop hook even when @@ -840,9 +870,7 @@ test('enters stopping-graceful and runs the stop hook even with no pending commi expect(outputs(h, 'reader.partition.stopped')).toHaveLength(0) expect(h.effects).toContainEqual({ type: 'reader.effect.partition.stop_hook', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId, + session, committedOffset: 5n, }) expect(h.effects).toContainEqual({ @@ -903,7 +931,7 @@ test('rejects a commit that lost the race with the partition stop', () => { expect(String((rejected[0] as { reason?: unknown }).reason)).toMatch( /stopped or expired partition session/ ) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toEqual([]) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toEqual([]) }) // The same race against a forced stop: offsets the stop's watermark already covers @@ -921,6 +949,35 @@ test('settles a commit racing a forced stop against the stop watermark', () => { expect(outputs(h, 'reader.commit.rejected')[0]).toMatchObject({ waiterId: 9 }) }) +test('forgets a stopped partition after queued commits consume its watermark', () => { + let h = mk() + toReadyWithPartition(h) + ackStart(h, 1n, 10n) + message(h, stopMsg(1n, false, 7n)) + let cleanup = h.dispatched.find((event) => event.type === 'reader.partition.forget')! + expect(cleanup).toBeDefined() + commit(h, 10n, [{ start: 5n, end: 7n }], 8) + expect(outputs(h, 'reader.commit.resolved')).toContainEqual({ + type: 'reader.commit.resolved', + waiterId: 8, + }) + step(h, cleanup) + expect(h.ctx.partitions.has(pk(10n))).toBe(false) +}) + +test('keeps a regranted partition when cleanup for its old session runs', () => { + let h = mk() + toReadyWithPartition(h) + message(h, stopMsg(1n, false)) + let cleanup = h.dispatched.find((event) => event.type === 'reader.partition.forget')! + message(h, startMsg(1n, 10n, 5n)) + let current = h.ctx.partitions.get(pk(10n))!.session + step(h, cleanup) + expect(h.ctx.partitions.get(pk(10n))!.session).toBe(current) + ackStart(h, 1n, 10n) + expect(startResponses(h.effects)).toHaveLength(1) +}) + test('resolves covered commits and holds the remainder on a forced stop', () => { let h = mk() toReadyWithPartition(h) @@ -934,7 +991,7 @@ test('resolves covered commits and holds the remainder on a forced stop', () => expect(outputs(h, 'reader.commit.resolved').map((o) => o.waiterId)).toEqual([1]) let entry = h.ctx.partitions.get(pk(10n))! expect(entry.state).toBe('stopped') - expect(entry.pendingCommits.map((p) => p.waiterId)).toEqual([2]) + expect(entry.commitWaiters.map((p) => p.waiterId)).toEqual([2]) expect(h.effects).toContainEqual({ type: 'reader.effect.timer.schedule', which: 'partition_reassign_gc', @@ -1063,7 +1120,7 @@ test('sends exactly one stop_response when the graceful timer wins the race', () test('drops a late stop_ready after a force stop escalates the graceful stop', () => { let h = mk() toReadyWithPartition(h) - let grantId = h.ctx.partitions.get(pk(10n))!.grantId + let session = h.ctx.partitions.get(pk(10n))!.session message(h, stopMsg(1n, true)) h.emitted.length = 0 message(h, stopMsg(1n, false)) @@ -1075,15 +1132,15 @@ test('drops a late stop_ready after a force stop escalates the graceful stop', ( }) // The stop hook completes after the escalation already gave the partition up: a // stop_response for the released session is session-fatal. - step(h, { type: 'reader.partition.stop_ready', partitionKey: pk(10n), grantId }) + step(h, { type: 'reader.partition.stop_ready', session }) expect(h.effects).toEqual([]) expect(stopResponses(h.allEffects)).toHaveLength(0) }) -test('ignores a stop_ready carrying a superseded grant id', () => { +test('ignores a stop_ready carrying a superseded session object', () => { let h = mk() toReadyWithPartition(h) - let staleGrant = h.ctx.partitions.get(pk(10n))!.grantId + let staleSession = h.ctx.partitions.get(pk(10n))!.session message(h, stopMsg(1n, true)) // stop hook for the first grant starts running step(h, { type: 'reader.stream.disconnected', @@ -1091,11 +1148,17 @@ test('ignores a stop_ready carrying a superseded grant id', () => { }) step(h, { type: 'reader.timer.retry_backoff' }) step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) - message(h, startMsg(2n, 10n, 5n)) // fresh grant supersedes the stopping one + message(h, startMsg(1n, 10n, 5n)) // the new stream reuses the wire id + expect(h.ctx.partitions.get(pk(10n))!.session).not.toBe(staleSession) + ackStart(h, 1n, 10n) + message(h, stopMsg(1n, true)) // The old grant's stop hook completes: it must not stop the fresh grant. - step(h, { type: 'reader.partition.stop_ready', partitionKey: pk(10n), grantId: staleGrant }) - expect(h.ctx.partitions.get(pk(10n))!.state).toBe('active') + step(h, { type: 'reader.partition.stop_ready', session: staleSession }) + expect(h.ctx.partitions.get(pk(10n))!.state).toBe('stopping-graceful') expect(stopResponses(h.allEffects)).toHaveLength(0) + ackStop(h, 10n) + expect(h.ctx.partitions.get(pk(10n))!.state).toBe('stopped') + expect(stopResponses(h.allEffects)).toHaveLength(1) }) test('sends the stop response when the graceful timeout fires on the live session', () => { @@ -1246,7 +1309,9 @@ test('ignores a late per-partition graceful timeout after the reader closed', () test('replenishes exactly the released read credit without delay', () => { let h = mk(1000n) toReadyWithPartition(h) - message(h, readMsg(1n, 300n, [5n])) + message(h, readMsg(1n, 150n, [5n])) + message(h, readMsg(1n, 100n, [6n])) + message(h, readMsg(1n, 50n, [7n])) h.effects = [] step(h, { type: 'reader.read_release', bytes: 150n }) let first = h.effects.find( @@ -1254,7 +1319,7 @@ test('replenishes exactly the released read credit without delay', () => { e.type === 'reader.effect.send.read_request' ) expect(first?.bytesSize).toBe(150n) - expect(h.ctx.inFlightBytes).toBe(150n) + expect(h.ctx.bufferedBytes).toBe(150n) h.effects = [] step(h, { type: 'reader.read_release', bytes: 100n }) let second = h.effects.find( @@ -1262,22 +1327,58 @@ test('replenishes exactly the released read credit without delay', () => { e.type === 'reader.effect.send.read_request' ) expect(second?.bytesSize).toBe(100n) - expect(h.ctx.inFlightBytes).toBe(50n) + expect(h.ctx.bufferedBytes).toBe(50n) }) -test('resets flow-control on reconnect init', () => { +test('reserves reconnect credit for retained responses', () => { let h = mk(1000n) toReadyWithPartition(h) message(h, readMsg(1n, 400n, [5n])) - expect(h.ctx.inFlightBytes).toBe(400n) + expect(h.ctx.bufferedBytes).toBe(400n) step(h, { type: 'reader.stream.disconnected', error: new YDBError(StatusIds_StatusCode.UNAVAILABLE, []), }) step(h, { type: 'reader.timer.retry_backoff' }) step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) - expect(h.ctx.inFlightBytes).toBe(0n) + expect(h.ctx.bufferedBytes).toBe(400n) + expect(h.state).toBe('ready') + expect(h.effects).toContainEqual({ type: 'reader.effect.send.read_request', bytesSize: 600n }) expect(h.ctx.sessionIndex.size).toBe(0) + step(h, { type: 'reader.read_release', bytes: 400n }) + expect(h.ctx.bufferedBytes).toBe(0n) + expect(h.state).toBe('ready') + expect(h.effects).toEqual([{ type: 'reader.effect.send.read_request', bytesSize: 400n }]) +}) + +test('accounts retained-byte releases during reconnect backoff and init', () => { + let h = mk(1000n) + toReadyWithPartition(h) + message(h, readMsg(1n, 200n, [5n])) + message(h, readMsg(1n, 100n, [6n])) + message(h, readMsg(1n, 300n, [7n])) + step(h, { type: 'reader.stream.disconnected' }) + step(h, { type: 'reader.read_release', bytes: 200n }) + expect(h.ctx.bufferedBytes).toBe(400n) + expect(h.effects).toEqual([]) + step(h, { type: 'reader.timer.retry_backoff' }) + step(h, { type: 'reader.read_release', bytes: 100n }) + expect(h.ctx.bufferedBytes).toBe(300n) + expect(h.effects).toEqual([]) + step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) + expect(h.state).toBe('ready') + expect(h.effects).toContainEqual({ type: 'reader.effect.send.read_request', bytesSize: 700n }) +}) + +test('repays current-stream overdraw without increasing the credit window', () => { + let h = mk(1000n) + toReadyWithPartition(h) + message(h, readMsg(1n, 1400n, [5n])) + expect(h.state).toBe('ready') + step(h, { type: 'reader.read_release', bytes: 1400n }) + expect(h.ctx.bufferedBytes).toBe(0n) + expect(h.state).toBe('ready') + expect(h.effects).toEqual([{ type: 'reader.effect.send.read_request', bytesSize: 1400n }]) }) // (tx read-offset tracking lives in the facade — the FSM is tx-agnostic; covered @@ -1301,7 +1402,7 @@ test('answers a grant exactly once when a stale start_ready from the previous st step(h, { type: 'reader.start' }) step(h, { type: 'reader.stream.init_response', sessionId: 's1' }) message(h, startMsg(1n, 10n, 5n)) - let staleGrant = h.ctx.partitions.get(pk(10n))!.grantId // hook 1 still running + let staleSession = h.ctx.partitions.get(pk(10n))!.session // hook 1 still running step(h, { type: 'reader.stream.disconnected', error: new YDBError(StatusIds_StatusCode.UNAVAILABLE, []), @@ -1309,17 +1410,16 @@ test('answers a grant exactly once when a stale start_ready from the previous st step(h, { type: 'reader.timer.retry_backoff' }) step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) // The new stream grants the same partition under the SAME per-stream id 1 (server - // assign ids restart at 1) — only the grantId epoch tells the two hooks apart. + // assign ids restart at 1) — the distinct session objects tell the two hooks apart. message(h, startMsg(1n, 10n, 5n)) + expect(h.ctx.partitions.get(pk(10n))!.session).not.toBe(staleSession) let before = h.allEffects.length // The stale hook (stream 1) completes, then the current one. A second // StartPartitionSessionResponse for one assign id is session-fatal // ("double partition locking", BAD_REQUEST). step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: staleGrant, + session: staleSession, }) ackStart(h, 1n, 10n) expect(startResponses(h.allEffects.slice(before))).toHaveLength(1) @@ -1330,9 +1430,7 @@ test('passes readOffset and commitOffset overrides through to the start response toReadyWithPartition(h) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, readOffset: 12n, commitOffset: 10n, }) @@ -1348,9 +1446,7 @@ test('confirms a commitOffset override only from a server watermark', () => { h.emitted.length = 0 step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) expect(outputs(h, 'reader.partition.committed')).toHaveLength(0) @@ -1374,9 +1470,7 @@ test('clamps a subsequent commit at the commitOffset override', () => { toReadyWithPartition(h) // server says committed 5 step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) commit(h, 10n, [{ start: 5n, end: 13n }], 1) @@ -1401,9 +1495,7 @@ test('waits for confirmation of an override covering a pending commit', () => { // Re-sending [5,8) after the override would overlap the server's pending commit. step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 7n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) expect(commitSends(h.effects)).toHaveLength(0) @@ -1421,9 +1513,7 @@ test('retries status confirmation and stops querying a revoked partition', () => toReadyWithPartition(h) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) step(h, { type: 'reader.stream.partition_status', partitionSessionId: 1n, committedOffset: 5n }) @@ -1458,9 +1548,7 @@ test('replays ranges suppressed by an unconfirmed override after reconnect', () toReadyWithPartition(h) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) commit(h, 10n, [{ start: 5n, end: 8n }], 1) @@ -1492,9 +1580,7 @@ test('drains an unconfirmed start override before closing', () => { toReadyWithPartition(h) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) step(h, { type: 'reader.close' }) @@ -1515,9 +1601,7 @@ test('accepts a stop watermark as confirmation of the start override', () => { toReadyWithPartition(h) step(h, { type: 'reader.partition.start_ready', - partitionSessionId: 1n, - partitionKey: pk(10n), - grantId: h.ctx.partitions.get(pk(10n))!.grantId, + session: h.ctx.partitions.get(pk(10n))!.session, commitOffset: 10n, }) message(h, stopMsg(1n, true, 10n)) @@ -1554,7 +1638,7 @@ test('does not double-send a commit issued between start_partition and start_rea // one send per pending with its own ranges. ackStart(h, 7n, 10n) let cs = commitSends(h.effects) - expect(cs.map((e) => e.ranges)).toEqual([[{ start: 5n, end: 8n }], [{ start: 8n, end: 10n }]]) + expect(cs.map((e) => e.ranges)).toEqual([[{ start: 5n, end: 10n }]]) }) test('keeps the reassign gc armed from reconnect until the start_ready ack clears it', () => { @@ -1591,6 +1675,27 @@ test('keeps the reassign gc armed from reconnect until the start_ready ack clear }) }) +test('bounds a commit submitted after reconnect when its partition is not reassigned', () => { + let h = mk() + toReadyWithPartition(h) + ackStart(h, 1n, 10n) + let session = h.ctx.partitions.get(pk(10n))!.session + step(h, { type: 'reader.stream.disconnected' }) + step(h, { type: 'reader.timer.retry_backoff' }) + step(h, { type: 'reader.stream.init_response', sessionId: 's2' }) + let cleanup = h.effects.filter( + (effect) => + effect.type === 'reader.effect.timer.schedule' && + effect.which === 'partition_reassign_gc' + ) + commit(h, 10n, [{ start: 5n, end: 6n }], 1) + expect(cleanup).toHaveLength(1) + step(h, { type: 'reader.timer.partition_reassign_gc', partitionKey: pk(10n) }) + expect(outputs(h, 'reader.commit.rejected').map((output) => output.waiterId)).toEqual([1]) + expect(h.ctx.partitions.has(pk(10n))).toBe(false) + expect(session.isStopped).toBe(true) +}) + test('bounds pending commits when the start hook never completes', () => { let h = mk() toReadyWithPartition(h) @@ -1610,8 +1715,8 @@ test('bounds pending commits when the start hook never completes', () => { expect(outputs(h, 'reader.commit.rejected').map((o) => o.waiterId)).toEqual([1]) // The granted entry survives so a late ack can still answer the server. expect(h.ctx.partitions.has(pk(10n))).toBe(true) - expect(h.ctx.partitions.get(pk(10n))!.pendingCommits).toHaveLength(0) - expect(h.ctx.partitions.get(pk(10n))!.claimedRanges).toHaveLength(0) + expect(h.ctx.partitions.get(pk(10n))!.commitWaiters).toHaveLength(0) + expect(h.ctx.partitions.get(pk(10n))!.pendingRanges).toHaveLength(0) }) test('reaps a never-re-granted partition on gc and recreates it cleanly on a late grant', () => { @@ -1633,10 +1738,10 @@ test('reaps a never-re-granted partition on gc and recreates it cleanly on a lat // A LATE grant after the reap creates a fresh entry. message(h, startMsg(9n, 10n, 20n)) let entry = h.ctx.partitions.get(pk(10n))! - expect(entry.partitionSessionId).toBe(9n) + expect(entry.session.partitionSessionId).toBe(9n) expect(entry.state).toBe('active') expect(entry.deliveredWatermark).toBe(20n) - expect(entry.pendingCommits).toHaveLength(0) + expect(entry.commitWaiters).toHaveLength(0) ackStart(h, 9n, 10n) expect(startResponses(h.effects)).toHaveLength(1) }) @@ -1661,8 +1766,10 @@ test('rejects outstanding commits exactly once on destroy and terminates', () => toReadyWithPartition(h) commit(h, 10n, [{ start: 5n, end: 9n }], 1) h.emitted.length = 0 - step(h, { type: 'reader.destroy', reason: new Error('boom') }) + let reason = new Error('boom') + step(h, { type: 'reader.destroy', reason }) expect(h.state).toBe('closed') + expect(h.effects).toEqual([{ type: 'reader.effect.finalize', reason }]) expect(outputs(h, 'reader.commit.rejected').filter((r) => r.waiterId === 1)).toHaveLength(1) expect(outputs(h, 'reader.closed')).toHaveLength(1) expect(h.ctx.partitions.size).toBe(0) @@ -1673,6 +1780,7 @@ test('closes immediately when nothing is pending', () => { toReadyWithPartition(h) step(h, { type: 'reader.close' }) expect(h.state).toBe('closed') + expect(effectTypes(h.effects)).toEqual(['reader.effect.finalize']) }) test('waits for pending commits before closing', () => { @@ -1745,3 +1853,63 @@ test('finalizes from closing when the reassign gc drains the last held commit', step(h, { type: 'reader.timer.partition_reassign_gc', partitionKey: pk(10n) }) expect(h.state).toBe('closed') }) + +test('drains commits and refreshes the token before retained responses allow the first read request', () => { + let h = mk(1000n) + toReadyWithPartition(h) + ackStart(h, 1n, 10n) + message(h, readMsg(1n, 1000n, [5n, 6n])) + commit(h, 10n, [{ start: 5n, end: 6n }], 1) + step(h, { type: 'reader.stream.disconnected' }) + step(h, { type: 'reader.timer.retry_backoff' }) + step(h, { type: 'reader.stream.init_response', sessionId: '' }) + expect(h.state).toBe('waiting-credit') + expect(outputs(h, 'reader.session')).toContainEqual({ type: 'reader.session', sessionId: '' }) + expect(effectTypes(h.effects)).not.toContain('reader.effect.send.read_request') + message(h, startMsg(2n, 10n, 5n)) + ackStart(h, 2n, 10n) + expect(commitSends(h.effects)).toEqual([ + { + type: 'reader.effect.send.commit', + partitionSessionId: 2n, + ranges: [{ start: 5n, end: 6n }], + }, + ]) + message(h, commitMsg([[2n, 6n]])) + expect(outputs(h, 'reader.commit.resolved')).toContainEqual({ + type: 'reader.commit.resolved', + waiterId: 1, + }) + step(h, { type: 'reader.timer.update_token' }) + expect(h.effects).toEqual([{ type: 'reader.effect.send.update_token' }]) + step(h, { type: 'reader.timer.start_timeout' }) + expect(h.state).toBe('waiting-credit') + expect(h.effects).toEqual([]) + commit(h, 10n, [{ start: 6n, end: 7n }], 2) + expect(commitSends(h.effects)).toHaveLength(1) + step(h, { type: 'reader.close' }) + expect(h.state).toBe('closing') + message(h, commitMsg([[2n, 7n]])) + expect(h.state).toBe('closed') + expect(outputs(h, 'reader.commit.resolved')).toContainEqual({ + type: 'reader.commit.resolved', + waiterId: 2, + }) +}) + +test('distinguishes receive-size failures from temporary resource exhaustion', () => { + expect( + isRetryableTopicError( + new ClientError( + '/read', + Status.RESOURCE_EXHAUSTED, + 'Received message larger than max (2097152 vs. 1048576)' + ) + ) + ).toBe(false) + expect( + isRetryableTopicError( + new ClientError('/read', Status.RESOURCE_EXHAUSTED, 'Rate limit exceeded') + ) + ).toBe(true) +}) diff --git a/packages/topic/src/reader/reader-state.ts b/packages/topic/src/reader/reader-state.ts index 2bb904fc..203b0455 100644 --- a/packages/topic/src/reader/reader-state.ts +++ b/packages/topic/src/reader/reader-state.ts @@ -1,14 +1,13 @@ import type { Timestamp } from '@bufbuild/protobuf/wkt' -import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { loggers } from '@ydbjs/debug' -import { YDBError } from '@ydbjs/error' import type { TransitionResult, TransitionRuntime } from '@ydbjs/fsm' -import { isRetryableError, isRetryableStreamError } from '@ydbjs/retry' + +import { isRetryableTopicError } from '../retry.js' import { TopicPartitionSession } from '../partition-session.js' // The pure half of the reader: states, context, and a synchronous transition -// with no I/O. Mirrors writer-state.ts. The transport FSM owns one streamRead +// with no I/O. Mirrors writer-state.ts. The transport owns one streamRead // stream and forwards server frames; the runtime's mapTransportOutput CLASSIFIES // them into the typed `reader.stream.*` events below, so this transition works on // clean domain events — never raw protobuf. @@ -16,7 +15,7 @@ import { TopicPartitionSession } from '../partition-session.js' // The full transition map (table + diagram) lives in packages/topic/ARCHITECTURE.md — // update it in the same commit when you change this dispatch. // -// KEY DESIGN (see project decisions): all reader state is keyed by the STABLE +// Partition state is keyed by the stable // (topicPath, partitionId) pair — partitionKey() — never the ephemeral // partitionSessionId (proto: "unique inside one RPC call"). partition_id alone is // per-topic, so a multi-topic reader routinely sees the same id from several topics. @@ -32,6 +31,7 @@ let dbg = loggers.topic.extend('reader') export type ReaderState = | 'idle' | 'connecting' + | 'waiting-credit' | 'ready' | 'reconnecting' | 'closing' @@ -47,30 +47,17 @@ export let partitionKey = function partitionKey(path: string, partitionId: bigin return `${path}|${partitionId}` } -// A commit awaiting the server's high-water-mark ack. `targetOffset` controls promise -// completion; `wireRanges` is only the unshared coverage this commit put (or will put) -// on the wire. A fully claimed duplicate has no wire ranges but still waits for the -// server watermark instead of resolving optimistically. -export type PendingCommit = { +// Every commit call waits for the server watermark, including repeated requests +// for ranges already pending on the partition. +export type CommitWaiter = { targetOffset: bigint - wireRanges: OffsetRange[] waiterId: number } // Everything the reader tracks about one partition, keyed by the stable partitionKey. export type PartitionEntry = { - // identity — stable across reconnects - partitionId: bigint - path: string - - // current ephemeral grant (replaced on each reconnect/reassign) - partitionSessionId: bigint + // Each grant creates a new object, even when a new stream reuses the wire id. session: TopicPartitionSession - // Identity of the latest grant (start_partition), stamped through the async - // onPartitionSessionStart handshake: partition session ids restart at 1 per - // stream, so without it a hook completing after a reconnect could answer a NEW - // grant that reused the id — the server kills the session for a double response. - grantId: number // True from the grant until its start_ready is honored. While set, commits are // buffered (the ack performs the single send — a commit must never hit the wire // twice, nor before the start response), and the reassign gc treats the entry as @@ -78,7 +65,6 @@ export type PartitionEntry = { ackPending: boolean // offset tracking - partitionOffsets: OffsetRange partitionCommittedOffset: bigint // A start response requests this offset but does not acknowledge its durability. requestedCommitOffset: bigint | undefined @@ -88,12 +74,9 @@ export type PartitionEntry = { // (retention, readFrom skips) are attributed to the NEXT delivered message — // and a delivered-but-unacked message is never covered by another message's range. deliveredWatermark: bigint - // Offsets already claimed by a sent/buffered commit. Re-sending an overlapping - // or rewound range is session-fatal (BAD_REQUEST "double committing is - // forbidden"), so every new commit subtracts this coverage first. Rebuilt from - // the narrowed pending commits on each grant; compacted as the server watermark - // advances. - claimedRanges: OffsetRange[] + // Requested ranges not yet covered by the server watermark. They belong to the + // partition, independently of how many commit promises await their confirmation. + pendingRanges: OffsetRange[] // lifecycle state: 'active' | 'stopping-graceful' | 'stopped' | 'ended' @@ -101,7 +84,7 @@ export type PartitionEntry = { // completed; the stop response goes out only when stopReady AND the pending // commits drained (either order), or the per-partition timeout escalates. stopReady: boolean - pendingCommits: PendingCommit[] + commitWaiters: CommitWaiter[] } // One partition's slice of a ReadResponse — the structural shape the transition @@ -148,10 +131,6 @@ export type ReaderLimits = { // Pure logical context — mutated synchronously inside the transition only. export type ReaderCtx = { - // connection identity - sessionId: string - hasEverConnected: boolean - // reconnect bookkeeping (mirrors the writer) attempts: number lastError: unknown @@ -166,11 +145,9 @@ export type ReaderCtx = { partitions: Map // ephemeral partitionSessionId -> partitionKey, rebuilt each stream sessionIndex: Map - // monotonic grant counter — source of PartitionEntry.grantId - grantSeq: number - // byte flow-control - inFlightBytes: bigint + // Unreleased ReadResponse.bytesSize, including responses retained across reconnects. + bufferedBytes: bigint limits: ReaderLimits } @@ -239,18 +216,16 @@ export type ReaderEvent = | { type: 'reader.read_release'; bytes: bigint } | { type: 'reader.close' } | { type: 'reader.destroy'; reason?: unknown } + | { type: 'reader.partition.forget'; session: TopicPartitionSession } // transport -> reader | { type: 'reader.stream.init_response'; sessionId: string } | ReaderStreamEvent | { type: 'reader.stream.disconnected'; error?: unknown } // The async onPartitionSessionStart hook finished (runs detached in the runtime so - // user code never blocks the drain loop); carries the hook's offset overrides and - // the grantId of the grant it answers (guards against stale completions). + // user code never blocks the drain loop); carries the exact session it answers. | { type: 'reader.partition.start_ready' - partitionSessionId: bigint - partitionKey: string - grantId: number + session: TopicPartitionSession readOffset?: bigint commitOffset?: bigint } @@ -258,8 +233,7 @@ export type ReaderEvent = // response may go out once the pending commits drain too. | { type: 'reader.partition.stop_ready' - partitionKey: string - grantId: number + session: TopicPartitionSession } // timers | { type: 'reader.timer.start_timeout' } @@ -292,13 +266,10 @@ export type ReaderEffect = commitOffset?: bigint } | { type: 'reader.effect.send.update_token' } - | { type: 'reader.effect.transport.close' } // Runs the async onPartitionSessionStart callback then sends the response. | { type: 'reader.effect.partition.start_hook' - partitionSessionId: bigint - partitionKey: string - grantId: number + session: TopicPartitionSession committedOffset: bigint partitionOffsets: OffsetRange } @@ -307,9 +278,7 @@ export type ReaderEffect = // the hook runs — this is the documented "last chance to commit". | { type: 'reader.effect.partition.stop_hook' - partitionSessionId: bigint - partitionKey: string - grantId: number + session: TopicPartitionSession committedOffset: bigint } | ({ type: 'reader.effect.timer.schedule' } & TimerRef) @@ -360,9 +329,6 @@ export let createReaderCtx = function createReaderCtx( options?: { retryOnSchemeError?: boolean; recoveryWindowMs?: number } ): ReaderCtx { return { - sessionId: '', - hasEverConnected: false, - attempts: 0, lastError: undefined, retryOnSchemeError: options?.retryOnSchemeError ?? false, @@ -370,36 +336,13 @@ export let createReaderCtx = function createReaderCtx( partitions: new Map(), sessionIndex: new Map(), - grantSeq: 0, - inFlightBytes: 0n, + bufferedBytes: 0n, limits, } } -// Reconnecting is always safe for a reader (offsets are server-tracked), so we -// retry any retryable stream error; a clean end (undefined) is a server-side -// reconnect. Fatal statuses (SCHEME_ERROR, UNAUTHORIZED, …) stop the reader — -// except SCHEME_ERROR is retried when `retryOnSchemeError` is set (wait for the -// topic to be created). -export let isRetryableReaderError = function isRetryableReaderError( - error: unknown, - retryOnSchemeError = false -): boolean { - if (error === undefined || error === null) { - return true - } - if ( - retryOnSchemeError && - error instanceof YDBError && - error.code === StatusIds_StatusCode.SCHEME_ERROR - ) { - return true - } - return isRetryableStreamError(error) || isRetryableError(error, false) -} - let clearConnectTimersEffects: ReaderEffect[] = [ { type: 'reader.effect.timer.clear', which: 'start_timeout' }, { type: 'reader.effect.timer.clear', which: 'retry_backoff' }, @@ -498,21 +441,19 @@ let compactCoverage = function compactCoverage( // waiterIds to resolve. Used by both the commit-ack path and the reconnect reconcile. let drainCommits = function drainCommits(entry: PartitionEntry, committedOffset: bigint): number[] { let resolved: number[] = [] - let kept: PendingCommit[] = [] - for (let pending of entry.pendingCommits) { + let kept: CommitWaiter[] = [] + for (let pending of entry.commitWaiters) { if (pending.targetOffset <= committedOffset) { resolved.push(pending.waiterId) } else { kept.push(pending) } } - entry.pendingCommits = kept + entry.commitWaiters = kept return resolved } -// Re-grant reconcile: narrow every pending's wire ranges to server truth, resolve by -// target watermark, and rebuild the claimed coverage from what is left. Empty wire -// ranges remain pending when another in-flight commit owns their coverage. +// Re-grant reconcile preserves only unconfirmed coverage and pending waiters. let narrowPendings = function narrowPendings( entry: PartitionEntry, committedOffset: bigint, @@ -521,10 +462,7 @@ let narrowPendings = function narrowPendings( for (let waiterId of drainCommits(entry, committedOffset)) { runtime.emit({ type: 'reader.commit.resolved', waiterId }) } - for (let pending of entry.pendingCommits) { - pending.wireRanges = compactCoverage(pending.wireRanges, committedOffset) - } - entry.claimedRanges = mergeRanges(entry.pendingCommits.flatMap((pending) => pending.wireRanges)) + entry.pendingRanges = compactCoverage(entry.pendingRanges, committedOffset) } // Advance the server-confirmed committed watermark; emits partition.committed (the @@ -540,13 +478,13 @@ let advanceCommitted = function advanceCommitted( } entry.partitionCommittedOffset = committedOffset entry.session.partitionCommittedOffset = committedOffset - entry.claimedRanges = compactCoverage(entry.claimedRanges, committedOffset) + entry.pendingRanges = compactCoverage(entry.pendingRanges, committedOffset) if (entry.deliveredWatermark < committedOffset) { entry.deliveredWatermark = committedOffset } runtime.emit({ type: 'reader.partition.committed', - partitionId: entry.partitionId, + partitionId: entry.session.partitionId, committedOffset, session: entry.session, }) @@ -607,7 +545,7 @@ let readResponse = function readResponse( // belong to the app — the soft-stop contract is "finish processing and commit". if ( !entry || - entry.partitionSessionId !== partitionData.partitionSessionId || + entry.session.partitionSessionId !== partitionData.partitionSessionId || entry.state === 'stopped' || entry.state === 'ended' ) { @@ -621,7 +559,7 @@ let readResponse = function readResponse( // Charge the whole response once and emit a single batch so the consumer releases // exactly bytesSize — several partitions in one response must not each claim the // full size. Emit even when everything was dropped so credit is still released. - ctx.inFlightBytes += event.bytesSize + ctx.bufferedBytes += event.bytesSize runtime.emit({ type: 'reader.messages', releaseBytes: event.bytesSize, groups }) return [] } @@ -634,40 +572,31 @@ let upsertPartitionEntry = function upsertPartitionEntry( event: Extract, session: TopicPartitionSession ): PartitionEntry { - let { partitionSessionId, partitionId, path, committedOffset, partitionOffsets } = event + let { partitionSessionId, partitionId, path, committedOffset } = event let key = partitionKey(path, partitionId) let entry = ctx.partitions.get(key) - ctx.grantSeq += 1 if (entry === undefined) { entry = { - partitionId, - path, - - partitionSessionId, session, - grantId: ctx.grantSeq, ackPending: true, - partitionOffsets, partitionCommittedOffset: committedOffset, requestedCommitOffset: undefined, deliveredWatermark: committedOffset, - claimedRanges: [], + pendingRanges: [], state: 'active', stopReady: false, - pendingCommits: [], + commitWaiters: [], } ctx.partitions.set(key, entry) } else { // Reconnect/reassign: drop the superseded session id from the index (so a late // message on the dead session can never misroute), install the fresh session // object + id, keep pending commits for the reconcile in startPartitionSession. - ctx.sessionIndex.delete(entry.partitionSessionId) + ctx.sessionIndex.delete(entry.session.partitionSessionId) entry.session = session - entry.partitionSessionId = partitionSessionId - entry.partitionOffsets = partitionOffsets entry.requestedCommitOffset = undefined // Server truth may only move the committed mark forward. if (entry.partitionCommittedOffset < committedOffset) { @@ -677,7 +606,6 @@ let upsertPartitionEntry = function upsertPartitionEntry( entry.deliveredWatermark = entry.partitionCommittedOffset entry.state = 'active' entry.stopReady = false - entry.grantId = ctx.grantSeq entry.ackPending = true } ctx.sessionIndex.set(partitionSessionId, key) @@ -727,17 +655,28 @@ let startPartitionSession = function startPartitionSession( // onPartitionSessionStart callback, so the runtime sends it from an effect. effects.push({ type: 'reader.effect.partition.start_hook', - partitionSessionId, - partitionKey: key, - grantId: entry.grantId, + session, committedOffset, partitionOffsets, }) return effects } -// Retire a partition session: mark it stopped, drop it from the ephemeral index, -// and notify the facade. Pending commits stay on the entry for reconcile/gc. +let queuePartitionCleanup = function queuePartitionCleanup( + entry: PartitionEntry, + runtime: ReaderRuntime +): void { + if ( + entry.commitWaiters.length === 0 && + entry.pendingRanges.length === 0 && + entry.requestedCommitOffset === undefined + ) { + // Commits queued behind the stop can still be covered by its watermark. + runtime.dispatch({ type: 'reader.partition.forget', session: entry.session }) + } +} + +// Pending commits stay on the stopped entry until reconciliation completes. let markStopped = function markStopped( ctx: ReaderCtx, entry: PartitionEntry, @@ -747,13 +686,14 @@ let markStopped = function markStopped( entry.session.stop() entry.state = 'stopped' entry.requestedCommitOffset = undefined - ctx.sessionIndex.delete(entry.partitionSessionId) + ctx.sessionIndex.delete(entry.session.partitionSessionId) runtime.emit({ type: 'reader.partition.stopped', - partitionId: entry.partitionId, + partitionId: entry.session.partitionId, reason, session: entry.session, }) + queuePartitionCleanup(entry, runtime) } let stopPartitionSession = function stopPartitionSession( @@ -798,7 +738,7 @@ let stopPartitionSession = function stopPartitionSession( partitionKey: key, }, ] - if (entry.pendingCommits.length > 0) { + if (entry.commitWaiters.length > 0) { effects.push({ type: 'reader.effect.timer.schedule', which: 'partition_reassign_gc', @@ -818,9 +758,7 @@ let stopPartitionSession = function stopPartitionSession( let effects: ReaderEffect[] = [ { type: 'reader.effect.partition.stop_hook', - partitionSessionId: event.partitionSessionId, - partitionKey: key, - grantId: entry.grantId, + session: entry.session, committedOffset: entry.partitionCommittedOffset, }, { @@ -840,8 +778,8 @@ let stopPartitionSession = function stopPartitionSession( } // The async onPartitionSessionStart hook finished — answer the server, unless the -// grant was superseded while the hook ran: grantId pins the exact grant (session ids -// collide across reconnects — they restart at 1 per stream), and ackPending dedupes +// grant was superseded while the hook ran: object identity pins the exact grant +// despite wire ids being reused across streams, and ackPending dedupes // a double completion. Only now is the session fully live: the reassign gc is // released and the pending commits are sent, strictly AFTER the start response and // exactly once (recordCommit buffers while ackPending). @@ -849,13 +787,14 @@ let ackPartitionStart = function ackPartitionStart( ctx: ReaderCtx, event: Extract ): ReaderEffect[] { - let entry = ctx.partitions.get(event.partitionKey) - if (!entry || entry.grantId !== event.grantId || !entry.ackPending) { + let key = partitionKey(event.session.topicPath, event.session.partitionId) + let entry = ctx.partitions.get(key) + if (!entry || entry.session !== event.session || !entry.ackPending) { return [] } if ( entry.state !== 'active' || - ctx.sessionIndex.get(entry.partitionSessionId) !== event.partitionKey + ctx.sessionIndex.get(entry.session.partitionSessionId) !== key ) { return [] } @@ -870,32 +809,27 @@ let ackPartitionStart = function ackPartitionStart( let effects: ReaderEffect[] = [ { type: 'reader.effect.send.start_response', - partitionSessionId: event.partitionSessionId, + partitionSessionId: event.session.partitionSessionId, ...(event.readOffset !== undefined && { readOffset: event.readOffset }), ...(event.commitOffset !== undefined && { commitOffset: event.commitOffset }), }, { type: 'reader.effect.timer.clear', which: 'partition_reassign_gc', - partitionKey: event.partitionKey, + partitionKey: key, }, ] - // The single send of everything buffered for this partition (reconciled pendings - // from before the reconnect and commits issued during the hook window alike) — - // each pending replays its exact remaining ranges, never a gap-covering span. - for (let pending of entry.pendingCommits) { - let ranges = compactCoverage( - pending.wireRanges, - entry.requestedCommitOffset ?? entry.partitionCommittedOffset - ) - if (ranges.length > 0) { - effects.push(commitEffect(entry.partitionSessionId, ranges)) - } + let ranges = compactCoverage( + entry.pendingRanges, + entry.requestedCommitOffset ?? entry.partitionCommittedOffset + ) + if (ranges.length > 0) { + effects.push(commitEffect(entry.session.partitionSessionId, ranges)) } if (entry.requestedCommitOffset !== undefined) { effects.push({ type: 'reader.effect.send.partition_status', - partitionSessionId: entry.partitionSessionId, + partitionSessionId: entry.session.partitionSessionId, }) } return effects @@ -903,34 +837,35 @@ let ackPartitionStart = function ackPartitionStart( // The async onPartitionSessionStop hook finished — the graceful stop may be // acknowledged once the pending commits drain too. Guards mirror ackPartitionStart: -// grantId pins the grant, state pins the phase (a force stop or reconnect while the +// session identity pins the grant, state pins the phase (a force stop or reconnect while the // hook ran makes this a no-op). let ackPartitionStop = function ackPartitionStop( ctx: ReaderCtx, event: Extract, runtime: ReaderRuntime ): ReaderEffect[] { - let entry = ctx.partitions.get(event.partitionKey) - if (!entry || entry.grantId !== event.grantId || entry.state !== 'stopping-graceful') { + let key = partitionKey(event.session.topicPath, event.session.partitionId) + let entry = ctx.partitions.get(key) + if (!entry || entry.session !== event.session || entry.state !== 'stopping-graceful') { return [] } entry.stopReady = true - if (entry.pendingCommits.length > 0 || entry.requestedCommitOffset !== undefined) { + if (entry.commitWaiters.length > 0 || entry.requestedCommitOffset !== undefined) { return [] } - let sessionId = entry.partitionSessionId - let live = ctx.sessionIndex.get(sessionId) === event.partitionKey + let sessionId = entry.session.partitionSessionId + let live = ctx.sessionIndex.get(sessionId) === key markStopped(ctx, entry, 'graceful', runtime) let effects: ReaderEffect[] = [ { type: 'reader.effect.timer.clear', which: 'partition_commit_status', - partitionKey: event.partitionKey, + partitionKey: key, }, { type: 'reader.effect.timer.clear', which: 'partition_graceful_timeout', - partitionKey: event.partitionKey, + partitionKey: key, }, ] // Answer only over the stream that asked (session ids restart per stream). @@ -961,7 +896,7 @@ let forceStopStalledGraceful = function forceStopStalledGraceful( entry.requestedCommitOffset = undefined runtime.emit({ type: 'reader.partition.stopped', - partitionId: entry.partitionId, + partitionId: entry.session.partitionId, reason: 'graceful', session: entry.session, }) @@ -971,17 +906,18 @@ let forceStopStalledGraceful = function forceStopStalledGraceful( // partition — releasing it is session-fatal (BAD_REQUEST) and deleting its index // mapping would silently drop that partition's reads. A stale stop needs no // answer at all: the server re-requests it on the new session if still relevant. - if (ctx.sessionIndex.get(entry.partitionSessionId) === key) { - ctx.sessionIndex.delete(entry.partitionSessionId) - effects.push(stopResponseEffect(entry.partitionSessionId)) + if (ctx.sessionIndex.get(entry.session.partitionSessionId) === key) { + ctx.sessionIndex.delete(entry.session.partitionSessionId) + effects.push(stopResponseEffect(entry.session.partitionSessionId)) } - if (entry.pendingCommits.length > 0) { + if (entry.commitWaiters.length > 0) { effects.push({ type: 'reader.effect.timer.schedule', which: 'partition_reassign_gc', partitionKey: key, }) } + queuePartitionCleanup(entry, runtime) return effects } @@ -1019,7 +955,7 @@ let commitOffsetResponse = function commitOffsetResponse( entry.state === 'stopping-graceful' && entry.stopReady && entry.requestedCommitOffset === undefined && - entry.pendingCommits.length === 0 + entry.commitWaiters.length === 0 ) { markStopped(ctx, entry, 'graceful', runtime) effects.push(stopResponseEffect(committed.partitionSessionId), { @@ -1050,7 +986,7 @@ let endPartitionSession = function endPartitionSession( entry.state = 'ended' runtime.emit({ type: 'reader.partition.stopped', - partitionId: entry.partitionId, + partitionId: entry.session.partitionId, reason: 'ended', session: entry.session, }) @@ -1131,7 +1067,7 @@ let recordCommit = function recordCommit( // remains pending until the server confirms its target watermark. let wireRanges = subtractCovered( requestedRanges, - entry.claimedRanges, + entry.pendingRanges, entry.partitionCommittedOffset ) @@ -1152,11 +1088,11 @@ let recordCommit = function recordCommit( return [] } - entry.pendingCommits.push({ targetOffset, wireRanges, waiterId: event.waiterId }) - entry.claimedRanges = mergeRanges([...entry.claimedRanges, ...wireRanges]) + entry.commitWaiters.push({ targetOffset, waiterId: event.waiterId }) + entry.pendingRanges = mergeRanges([...entry.pendingRanges, ...wireRanges]) - // Send only in `ready` and only over a session granted by the CURRENT stream: - // toReady clears sessionIndex and only start_partition repopulates it, so a commit + // An initialized stream can commit while retained responses delay its first read request. + // initializeSession clears sessionIndex and only start_partition repopulates it, so a commit // landing in the init→start_partition window after a reconnect buffers here and // rides the reconcile — sending immediately would use the previous stream's // session id. @@ -1168,8 +1104,8 @@ let recordCommit = function recordCommit( // buffered too: the ack performs the single send — sending here AND there would // put the same range on the wire twice, which is session-fatal. let sessionLive = - runtime.state === 'ready' && - ctx.sessionIndex.get(entry.partitionSessionId) === event.partitionKey + (runtime.state === 'ready' || runtime.state === 'waiting-credit') && + ctx.sessionIndex.get(entry.session.partitionSessionId) === event.partitionKey let committable = entry.state === 'active' || entry.state === 'stopping-graceful' || entry.state === 'ended' if (sessionLive && committable && !entry.ackPending && wireRanges.length > 0) { @@ -1177,20 +1113,37 @@ let recordCommit = function recordCommit( wireRanges, entry.requestedCommitOffset ?? entry.partitionCommittedOffset ) - return ranges.length > 0 ? [commitEffect(entry.partitionSessionId, ranges)] : [] + return ranges.length > 0 ? [commitEffect(entry.session.partitionSessionId, ranges)] : [] } return [] } -// Replenish exactly the server-accounted size of responses that have fully passed -// through read(). This keeps the stream's credit window stable even when one response -// exceeds the initial grant. -let releaseBytes = function releaseBytes(ctx: ReaderCtx, bytes: bigint): ReaderEffect[] { - ctx.inFlightBytes -= bytes - if (ctx.inFlightBytes < 0n) { - ctx.inFlightBytes = 0n +let startReading = function startReading( + ctx: ReaderCtx +): TransitionResult { + let bytes = ctx.limits.maxBufferBytes - ctx.bufferedBytes + return bytes > 0n + ? { state: 'ready', effects: [readRequestEffect(bytes)] } + : { state: 'waiting-credit' } +} + +let releaseBytes = function releaseBytes( + ctx: ReaderCtx, + bytes: bigint, + state: ReaderState +): TransitionResult | void { + if (bytes <= 0n) { + return + } + let released = bytes < ctx.bufferedBytes ? bytes : ctx.bufferedBytes + ctx.bufferedBytes -= released + if (state === 'ready') { + // The facade releases each complete response once, using its server-supplied bytesSize. + return { effects: released > 0n ? [readRequestEffect(released)] : [] } + } + if (state === 'waiting-credit') { + return startReading(ctx) } - return bytes > 0n ? [readRequestEffect(bytes)] : [] } // ── Terminal / transitions ────────────────────────────────────────────────────── @@ -1208,10 +1161,10 @@ let terminate = function terminate( // Reject every outstanding commit exactly once — the facade settles the waiterId. for (let entry of ctx.partitions.values()) { - for (let pending of entry.pendingCommits) { + for (let pending of entry.commitWaiters) { runtime.emit({ type: 'reader.commit.rejected', waiterId: pending.waiterId, reason }) } - entry.pendingCommits = [] + entry.commitWaiters = [] } runtime.emit({ type: 'reader.closed', reason }) @@ -1222,64 +1175,45 @@ let terminate = function terminate( // Terminal: the runtime seals itself after the finalize effect runs, so the // buffered lifecycle outputs (reader.closed / rejects) are delivered first. final: { reason }, - effects: [ - { type: 'reader.effect.transport.close' }, - { type: 'reader.effect.timer.clear', which: 'start_timeout' }, - { type: 'reader.effect.timer.clear', which: 'retry_backoff' }, - { type: 'reader.effect.timer.clear', which: 'recovery_window' }, - { type: 'reader.effect.timer.clear', which: 'update_token' }, - { type: 'reader.effect.timer.clear', which: 'graceful_timeout' }, - { type: 'reader.effect.finalize', reason }, - ], + effects: [{ type: 'reader.effect.finalize', reason }], } } -let releaseState = function releaseState(ctx: ReaderCtx): void { +export let releaseState = function releaseState(ctx: ReaderCtx): void { ctx.partitions.clear() ctx.sessionIndex.clear() - ctx.inFlightBytes = 0n + ctx.bufferedBytes = 0n } -// Enter `ready` on a successful init. Unlike the writer there is no seqNo recovery: -// the server re-sends start_partition per partition, where reconcile happens. -let toReady = function toReady( +// Initialize the session before granting read credit; retained responses may delay +// the first read grant while partition control and commits already run. +let initializeSession = function initializeSession( ctx: ReaderCtx, sessionId: string, runtime: ReaderRuntime ): TransitionResult { - ctx.sessionId = sessionId - ctx.hasEverConnected = true ctx.attempts = 0 - - // Ephemeral session ids from the previous stream are dead; buffered ReadResponses - // on it are gone. Reset flow-control and re-issue the full initial credit (the new - // stream grants a fresh maxBufferBytes budget, so old pending credit is moot). ctx.sessionIndex.clear() - ctx.inFlightBytes = 0n runtime.emit({ type: 'reader.session', sessionId }) let effects: ReaderEffect[] = [ ...clearConnectTimersEffects, { type: 'reader.effect.timer.schedule', which: 'update_token' }, - readRequestEffect(ctx.limits.maxBufferBytes), ] - // Bound the wait for every partition holding pending commits: if the server does - // not re-grant it on this stream (rebalanced to another reader), the gc rejects - // the waiters instead of leaving them pending forever. start_partition clears the - // timer when the partition does come back. - for (let [key, entry] of ctx.partitions) { - if (entry.pendingCommits.length > 0) { - effects.push({ - type: 'reader.effect.timer.schedule', - which: 'partition_reassign_gc', - partitionKey: key, - }) - } + // A retained message can be committed after init. Bound every old grant until + // its start handshake completes, including grants with no pending commit yet. + for (let key of ctx.partitions.keys()) { + effects.push({ + type: 'reader.effect.timer.schedule', + which: 'partition_reassign_gc', + partitionKey: key, + }) } - return { state: 'ready', effects } + let reading = startReading(ctx) + return { ...reading, effects: [...effects, ...(reading.effects ?? [])] } } let toReconnecting = function toReconnecting( @@ -1288,7 +1222,7 @@ let toReconnecting = function toReconnecting( runtime: ReaderRuntime ): TransitionResult { // sessionIndex means "session ids granted by the CURRENT stream" — with the stream - // gone there are none. Clearing here (not only in toReady) keeps the guards in + // gone there are none. Clearing here (not only on init) keeps the guards in // recordCommit / forceStopStalledGraceful honest while connecting: nothing may be // sent under an id the next stream never granted. ctx.sessionIndex.clear() @@ -1344,7 +1278,7 @@ let toClosing = function toClosing( let hasPendingWork = function hasPendingWork(ctx: ReaderCtx): boolean { for (let entry of ctx.partitions.values()) { if ( - entry.pendingCommits.length > 0 || + entry.commitWaiters.length > 0 || entry.requestedCommitOffset !== undefined || entry.state === 'stopping-graceful' ) { @@ -1364,7 +1298,7 @@ let gcPartition = function gcPartition(ctx: ReaderCtx, key: string, runtime: Rea } // A granted-and-acked session is live — the ack cleared this timer, so a firing // against it is a stale race; ignore it. - if (ctx.sessionIndex.get(entry.partitionSessionId) === key && !entry.ackPending) { + if (ctx.sessionIndex.get(entry.session.partitionSessionId) === key && !entry.ackPending) { return } // Two reapable cases: the partition was never re-granted on this stream @@ -1372,16 +1306,17 @@ let gcPartition = function gcPartition(ctx: ReaderCtx, key: string, runtime: Rea // completed (a hung onPartitionSessionStart hook) — either way the waiters would // hang until terminal close. Reject them; keep an un-acked granted entry so a // late start_ready can still answer the server (it just has nothing to re-send). - for (let pending of entry.pendingCommits) { + for (let pending of entry.commitWaiters) { runtime.emit({ type: 'reader.commit.rejected', waiterId: pending.waiterId, reason: new Error(`Partition ${key} reassigned before commit was acknowledged`), }) } - entry.pendingCommits = [] - entry.claimedRanges = [] - if (ctx.sessionIndex.get(entry.partitionSessionId) !== key) { + entry.commitWaiters = [] + entry.pendingRanges = [] + if (ctx.sessionIndex.get(entry.session.partitionSessionId) !== key) { + entry.session.stop() ctx.partitions.delete(key) } } @@ -1394,6 +1329,64 @@ let ignored = function ignored(state: ReaderState, event: ReaderEvent): void { dbg.log('ignoring %s in state %s', event.type, state) } +let sessionTransition = function sessionTransition( + ctx: ReaderCtx, + event: ReaderEvent, + runtime: ReaderRuntime +): TransitionResult | void { + let state = runtime.state + switch (event.type) { + case 'reader.stream.read_response': + case 'reader.stream.start_partition': + case 'reader.stream.stop_partition': + case 'reader.stream.commit_response': + case 'reader.stream.partition_status': + case 'reader.stream.end_partition': { + let effects = applyStreamEvent(ctx, event, runtime) + return { effects } + } + + case 'reader.commit': + return { effects: recordCommit(ctx, event, runtime) } + + case 'reader.partition.start_ready': { + let effects = ackPartitionStart(ctx, event) + return { effects } + } + + case 'reader.partition.stop_ready': { + let effects = ackPartitionStop(ctx, event, runtime) + return { effects } + } + + case 'reader.timer.update_token': + return { effects: [{ type: 'reader.effect.send.update_token' }] } + + case 'reader.timer.partition_reassign_gc': + gcPartition(ctx, event.partitionKey, runtime) + return + + // Fallback for a graceful stop whose hook or commits never completed: the + // server waits for the stop response indefinitely, so the client must not. + case 'reader.timer.partition_graceful_timeout': { + let effects = forceStopStalledGraceful(ctx, event.partitionKey, runtime) + return { effects } + } + + case 'reader.stream.disconnected': + if (!isRetryableTopicError(event.error, ctx.retryOnSchemeError)) { + return terminate(ctx, 'errored', event.error, runtime) + } + return toReconnecting(ctx, event.error, runtime) + + case 'reader.close': + return toClosing(ctx, runtime) + + default: + return ignored(state, event) + } +} + export let readerTransition = function readerTransition( ctx: ReaderCtx, event: ReaderEvent, @@ -1401,24 +1394,42 @@ export let readerTransition = function readerTransition( ): TransitionResult | void { let state = runtime.state + if (event.type === 'reader.partition.forget') { + let key = partitionKey(event.session.topicPath, event.session.partitionId) + let entry = ctx.partitions.get(key) + if ( + entry?.session === event.session && + entry.state === 'stopped' && + entry.commitWaiters.length === 0 && + entry.pendingRanges.length === 0 && + entry.requestedCommitOffset === undefined + ) { + ctx.partitions.delete(key) + } + return + } + // Global: hard destroy from any non-terminal state. if (state !== 'closed' && state !== 'errored' && event.type === 'reader.destroy') { return terminate(ctx, 'closed', event.reason ?? new Error('Reader destroyed'), runtime) } + if (state !== 'closed' && state !== 'errored' && event.type === 'reader.read_release') { + return releaseBytes(ctx, event.bytes, state) + } if ( - (state === 'ready' || state === 'closing') && + (state === 'ready' || state === 'closing' || state === 'waiting-credit') && event.type === 'reader.timer.partition_commit_status' ) { let entry = ctx.partitions.get(event.partitionKey) if ( entry?.requestedCommitOffset !== undefined && - ctx.sessionIndex.get(entry.partitionSessionId) === event.partitionKey + ctx.sessionIndex.get(entry.session.partitionSessionId) === event.partitionKey ) { return { effects: [ { type: 'reader.effect.send.partition_status', - partitionSessionId: entry.partitionSessionId, + partitionSessionId: entry.session.partitionSessionId, }, ], } @@ -1455,11 +1466,11 @@ export let readerTransition = function readerTransition( case 'reconnecting': { switch (event.type) { case 'reader.stream.init_response': - return toReady(ctx, event.sessionId, runtime) + return initializeSession(ctx, event.sessionId, runtime) case 'reader.commit': // Buffered for re-send on the next start_partition (recordCommit never - // sends outside `ready`). + // sends before the new session has initialized). return { effects: recordCommit(ctx, event, runtime) } case 'reader.stream.disconnected': @@ -1468,7 +1479,7 @@ export let readerTransition = function readerTransition( event.type === 'reader.stream.disconnected' ? event.error : undefined if ( event.type === 'reader.stream.disconnected' && - !isRetryableReaderError(error, ctx.retryOnSchemeError) + !isRetryableTopicError(error, ctx.retryOnSchemeError) ) { return terminate(ctx, 'errored', error, runtime) } @@ -1519,63 +1530,9 @@ export let readerTransition = function readerTransition( } } - case 'ready': { - switch (event.type) { - case 'reader.stream.read_response': - case 'reader.stream.start_partition': - case 'reader.stream.stop_partition': - case 'reader.stream.commit_response': - case 'reader.stream.partition_status': - case 'reader.stream.end_partition': { - let effects = applyStreamEvent(ctx, event, runtime) - return { effects } - } - - case 'reader.commit': - return { effects: recordCommit(ctx, event, runtime) } - - case 'reader.partition.start_ready': { - let effects = ackPartitionStart(ctx, event) - return { effects } - } - - case 'reader.partition.stop_ready': { - let effects = ackPartitionStop(ctx, event, runtime) - return { effects } - } - - case 'reader.read_release': { - let effects = releaseBytes(ctx, event.bytes) - return { effects } - } - - case 'reader.timer.update_token': - return { effects: [{ type: 'reader.effect.send.update_token' }] } - - case 'reader.timer.partition_reassign_gc': - gcPartition(ctx, event.partitionKey, runtime) - return - - // Fallback for a graceful stop whose hook or commits never completed: the - // server waits for the stop response indefinitely, so the client must not. - case 'reader.timer.partition_graceful_timeout': { - let effects = forceStopStalledGraceful(ctx, event.partitionKey, runtime) - return { effects } - } - - case 'reader.stream.disconnected': - if (!isRetryableReaderError(event.error, ctx.retryOnSchemeError)) { - return terminate(ctx, 'errored', event.error, runtime) - } - return toReconnecting(ctx, event.error, runtime) - - case 'reader.close': - return toClosing(ctx, runtime) - - default: - return ignored(state, event) - } - } + case 'ready': + case 'waiting-credit': + return sessionTransition(ctx, event, runtime) case 'closing': { switch (event.type) { diff --git a/packages/topic/src/reader/reader.autopartitioning.test.ts b/packages/topic/src/reader/reader.autopartitioning.test.ts index f7423166..e35c78f5 100644 --- a/packages/topic/src/reader/reader.autopartitioning.test.ts +++ b/packages/topic/src/reader/reader.autopartitioning.test.ts @@ -11,6 +11,7 @@ import { readResponse, settle, startPartitionSession, + stopPartitionSession, } from './reader.fixtures.ts' // Autopartitioning split flow over the wire: the InitRequest opt-in flag, and the @@ -310,3 +311,51 @@ test('delivers child data granted before the parent commit is acked', async (tc) stream.respond(commitOffsetResponse([{ partitionSessionId: 1n, committedOffset: 1n }])) await expect(commit).resolves.toBeUndefined() }) + +// A controlled stream can interleave EndPartition, forced Stop and already granted child data. +test('delivers a granted child after an ended parent is forcibly stopped', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { + topic: '/t', + consumer: 'c', + autoPartitioningSupport: true, + maxBufferBytes: 100n, + }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await waitForStartAck(stream, 1n) + stream.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: 11n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('parent') }], + }) + ) + stream.respond(endPartitionSession({ partitionSessionId: 1n, childPartitionIds: [1n] })) + stream.respond(startPartitionSession({ partitionSessionId: 2n, partitionId: 1n })) + await waitForStartAck(stream, 2n) + stream.respond( + readResponse({ + partitionSessionId: 2n, + bytesSize: 17n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('child') }], + }) + ) + stream.respond(stopPartitionSession({ partitionSessionId: 1n, graceful: false })) + await settle() + + let messages = await collectUpTo(reader, 1, tc.signal) + expect(messages.map((message) => text(message.payload))).toEqual(['child']) + let commit = reader.commit(messages) + let request = await stream.waitForCommit() + expect(request.commitOffsets[0]!.partitionSessionId).toBe(2n) + stream.respond(commitOffsetResponse([{ partitionSessionId: 2n, committedOffset: 1n }])) + await commit + await settle() + + let grants = stream.sent.flatMap((frame) => + frame.clientMessage.case === 'readRequest' ? [frame.clientMessage.value.bytesSize] : [] + ) + expect(grants.reduce((sum, grant) => sum + grant, 0n)).toBe(128n) + expect(reader.bufferedBytes).toBe(0n) +}) diff --git a/packages/topic/src/reader/reader.commit-semantics.test.ts b/packages/topic/src/reader/reader.commit-semantics.test.ts index ee6f4c1f..dacf00ce 100644 --- a/packages/topic/src/reader/reader.commit-semantics.test.ts +++ b/packages/topic/src/reader/reader.commit-semantics.test.ts @@ -636,3 +636,33 @@ test('invokes onCommittedOffset when a partition stop carries the committed wate expect(acks).toEqual([{ partitionId: 10n, committedOffset: 1n }]) }) + +test('handles a rejected async commit observer without delaying commit', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let observer = Promise.withResolvers() + using reader = createTopicReader(driver, { + topic: '/t', + consumer: 'c', + onCommittedOffset: async () => { + await observer.promise + throw new Error('Asynchronous commit observer failed') + }, + }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('message') }], + }) + ) + let messages = await collect(reader, 1, tc.signal) + let committed = reader.commit(messages) + await stream.waitForCommit() + stream.respond(commitOffsetResponse([{ partitionSessionId: 1n, committedOffset: 1n }])) + await expect(committed).resolves.toBeUndefined() + observer.resolve() + await new Promise((resolve) => setImmediate(resolve)) + await expect(reader.close()).resolves.toBeUndefined() +}) diff --git a/packages/topic/src/reader/reader.contract.test.ts b/packages/topic/src/reader/reader.contract.test.ts index af095002..f8a85a5f 100644 --- a/packages/topic/src/reader/reader.contract.test.ts +++ b/packages/topic/src/reader/reader.contract.test.ts @@ -1,10 +1,13 @@ +import { create } from '@bufbuild/protobuf' +import { DurationSchema, TimestampSchema } from '@bufbuild/protobuf/wkt' import { subscribe, unsubscribe } from 'node:diagnostics_channel' import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { Codec } from '@ydbjs/api/topic' import type { StreamReadMessage_FromServer } from '@ydbjs/api/topic' import { YDBError } from '@ydbjs/error' -import { expect, test } from 'vitest' +import { expect, test, vi } from 'vitest' +import { ClientError, Status } from 'nice-grpc' import { ZSTD_CODEC } from '../codec.ts' import type { TopicMessage } from '../message.ts' @@ -20,7 +23,6 @@ import { settle, startPartitionSession, stopPartitionSession, - updateTokenResponse, } from './reader.fixtures.ts' // End-to-end wiring of the reader facade against a fake streamRead: driver ↔ transport @@ -104,6 +106,13 @@ let primeStream = async function primeStream( return stream } +test.for([0n, -1n])('rejects an unusable read-credit window of %s bytes', (maxBufferBytes) => { + let { driver } = makeFakeTopicDriver() + expect(() => new TopicReader(driver, { topic: '/t', consumer: 'c', maxBufferBytes })).toThrow( + /maxBufferBytes must be positive/ + ) +}) + test('requests the full buffer as read credit after init', async () => { let { driver, waitForNextStream } = makeFakeTopicDriver() using reader = createTopicReader(driver, { @@ -199,7 +208,16 @@ test('resolves commit() when the server acknowledges the offset', async (tc) => await expect(commit).resolves.toBeUndefined() }) -test('does not reject commit() across a reconnect and resolves it on the new session', async (tc) => { +test.for([ + { name: 'clean end', error: undefined }, + { + name: 'server deadline', + error: new ClientError('/stream', Status.DEADLINE_EXCEEDED, 'deadline'), + }, + { name: 'server timeout', error: new YDBError(StatusIds_StatusCode.TIMEOUT, []) }, + { name: 'expired session', error: new YDBError(StatusIds_StatusCode.SESSION_EXPIRED, []) }, + { name: 'indeterminate result', error: new YDBError(StatusIds_StatusCode.UNDETERMINED, []) }, +])('preserves the pending commit across $name', async ({ error }, tc) => { let { driver, waitForNextStream } = makeFakeTopicDriver() using reader = createTopicReader(driver, { topic: '/t', consumer: 'c' }) @@ -222,10 +240,17 @@ test('does not reject commit() across a reconnect and resolves it on the new ses ) let messages = await collect(reader, 3, tc.signal) let commit = reader.commit(messages[2]!) + void commit.catch(() => {}) await a.waitForCommit() // The stream drops before the ack. The reader must NOT reject the in-flight commit. - a.disconnect() + if (error instanceof YDBError) { + a.respond(failureResponse(error.code)) + } else if (error) { + a.fail(error) + } else { + a.disconnect() + } // Session B: same partition, fresh session id, committed still 0. The reconcile must // re-send the pending [2, 3) verbatim on the new session id — never a widened span. @@ -1648,30 +1673,32 @@ test('keeps refreshing the token on the new stream after a reconnect', async () expect(update.token).toBe('fake-token') }) -test('coalesces update-token requests until one is acknowledged', async () => { - // The token interval fires on a schedule with no inflight gate. If the stream is - // open but the server never acks, un-coalesced pushes would pile token frames into - // the stream queue indefinitely (a slow long-lived leak). With coalescing, at most - // one un-acknowledged token is ever queued. +test('coalesces credential acquisition and refreshes without an ACK for the previous token', async () => { let { driver, waitForNextStream } = makeFakeTopicDriver() + let pending = Promise.withResolvers() + let token = vi + .spyOn(driver, 'token', 'get') + .mockReturnValueOnce(pending.promise) + .mockResolvedValue('renewed-token') using reader = createTopicReader(driver, { topic: '/t', consumer: 'c', - updateTokenIntervalMs: 5, // fire many times quickly + updateTokenIntervalMs: 5, }) - let stream = await primeStream(reader, waitForNextStream) - // Let the token interval fire many times without ever sending updateTokenResponse. await new Promise((resolve) => setTimeout(resolve, 60)) - - let tokenFrames = () => - stream.sent.filter((m) => m.clientMessage.case === 'updateTokenRequest').length - expect(tokenFrames()).toBe(1) - - // The ack clears the pending flag — the next tick sends a fresh refresh. - stream.respond(updateTokenResponse()) - await expect.poll(tokenFrames, { timeout: 5000 }).toBeGreaterThanOrEqual(2) + expect(token).toHaveBeenCalledTimes(1) + + pending.resolve('unchanged-token') + expect((await stream.waitForUpdateToken()).token).toBe('unchanged-token') + await expect + .poll(() => + stream.sent + .filter((message) => message.clientMessage.case === 'updateTokenRequest') + .map((message) => (message.clientMessage.value as { token: string }).token) + ) + .toContain('renewed-token') }) test('closes gracefully and aborts the underlying stream', async () => { @@ -1686,3 +1713,28 @@ test('closes gracefully and aborts the underlying stream', async () => { await settle() expect(stream.wasAborted()).toBe(true) }) + +test('keeps the configured source filters when caller-owned options change', async () => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let source = { + path: '/t', + partitionIds: [1n], + maxLag: create(DurationSchema, { seconds: 10n }), + readFrom: create(TimestampSchema, { seconds: 20n }), + } + using reader = createTopicReader(driver, { topic: source, consumer: 'c' }) + source.partitionIds.push(2n) + source.maxLag.seconds = 100n + source.readFrom.seconds = 200n + let first = await waitForNextStream() + let init = await first.waitForInit() + expect(init.topicsReadSettings).toMatchObject([ + { path: '/t', partitionIds: [1n], maxLag: { seconds: 10n }, readFrom: { seconds: 20n } }, + ]) + first.respond(initResponse('first')) + await first.waitForReadRequest() + first.disconnect() + let replacement = await waitForNextStream() + expect((await replacement.waitForInit()).topicsReadSettings).toEqual(init.topicsReadSettings) + await reader.close() +}) diff --git a/packages/topic/src/reader/reader.credit.test.ts b/packages/topic/src/reader/reader.credit.test.ts new file mode 100644 index 00000000..2affbb18 --- /dev/null +++ b/packages/topic/src/reader/reader.credit.test.ts @@ -0,0 +1,249 @@ +import { expect, test } from 'vitest' + +import { createTopicReader } from './index.ts' +import { + type FakeReadStream, + type FakeTopicDriver, + commitOffsetResponse, + initResponse, + makeFakeTopicDriver, + readResponse, + settle, + startPartitionSession, +} from './reader.fixtures.ts' + +// A fake stream makes reconnects land with retained responses or a pending init; +// these timing windows cannot be selected deterministically against real YDB. +let initStream = async function initStream(fake: FakeTopicDriver): Promise { + let stream = await fake.waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse()) + await settle() + return stream +} + +let credits = function credits(stream: FakeReadStream): bigint[] { + return stream.sent.flatMap((message) => + message.clientMessage.case === 'readRequest' ? [message.clientMessage.value.bytesSize] : [] + ) +} + +let send = function send(stream: FakeReadStream, offset: bigint, size: number): void { + stream.respond( + readResponse({ + partitionSessionId: 1n, + messages: [{ offset, seqNo: offset + 1n, data: new Uint8Array(size) }], + }) + ) +} + +test('keeps retained responses within one credit window across repeated reconnects', async (tc) => { + let fake = makeFakeTopicDriver() + using reader = createTopicReader(fake.driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 1000n, + }) + let stream = await initStream(fake) + expect(credits(stream)).toEqual([1000n]) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + send(stream, 0n, 1000) + await settle() + expect(reader.bufferedBytes).toBe(1000n) + + for (let i = 0; i < 3; i++) { + stream.disconnect() + // oxlint-disable-next-line no-await-in-loop + stream = await initStream(fake) + expect(credits(stream)).toEqual([]) + expect(reader.bufferedBytes).toBe(1000n) + } + + for await (let batch of reader.read({ signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([0n]) + expect(batch[0]!.payload.byteLength).toBe(1000) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(credits(stream)).toEqual([1000n]) + + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + send(stream, 1n, 1000) + for await (let batch of reader.read({ signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([1n]) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(credits(stream)).toEqual([1000n, 1000n]) +}) + +test('accounts releases while the next stream is waiting for init', async (tc) => { + let fake = makeFakeTopicDriver() + using reader = createTopicReader(fake.driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 1000n, + }) + let stream = await initStream(fake) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + send(stream, 0n, 400) + await settle() + stream.disconnect() + + let next = await fake.waitForNextStream() + await next.waitForInit() + for await (let batch of reader.read({ signal: tc.signal })) { + expect(batch).toHaveLength(1) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(credits(next)).toEqual([]) + + next.respond(initResponse()) + await settle() + expect(credits(next)).toEqual([1000n]) +}) + +test('withholds new credit until inherited oversized responses fit the buffer window', async (tc) => { + let fake = makeFakeTopicDriver() + using reader = createTopicReader(fake.driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 1000n, + }) + let stream = await initStream(fake) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + send(stream, 0n, 600) + send(stream, 1n, 800) + await settle() + expect(reader.bufferedBytes).toBe(1400n) + stream.disconnect() + let next = await initStream(fake) + expect(credits(next)).toEqual([]) + + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([0n]) + break + } + await settle() + expect(reader.bufferedBytes).toBe(800n) + expect(credits(next)).toEqual([200n]) + + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([1n]) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(credits(next)).toEqual([200n, 800n]) +}) + +test('shares reconnect credit between old responses and newly received data', async (tc) => { + let fake = makeFakeTopicDriver() + using reader = createTopicReader(fake.driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 1000n, + }) + let stream = await initStream(fake) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + send(stream, 0n, 400) + await settle() + stream.disconnect() + let next = await initStream(fake) + expect(credits(next)).toEqual([600n]) + next.respond(startPartitionSession({ partitionSessionId: 2n, partitionId: 1n })) + await next.waitForStartResponse() + next.respond( + readResponse({ + partitionSessionId: 2n, + messages: [{ offset: 0n, seqNo: 1n, data: new Uint8Array(600) }], + }) + ) + await settle() + expect(reader.bufferedBytes).toBe(1000n) + + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch[0]!.payload.byteLength).toBe(400) + break + } + await settle() + expect(reader.bufferedBytes).toBe(600n) + expect(credits(next)).toEqual([600n, 400n]) + next.respond( + readResponse({ + partitionSessionId: 2n, + messages: [{ offset: 1n, seqNo: 2n, data: new Uint8Array(400) }], + }) + ) + await settle() + expect(reader.bufferedBytes).toBe(1000n) + + for await (let batch of reader.read({ signal: tc.signal })) { + expect(batch[0]!.payload.byteLength).toBe(600) + break + } + for await (let batch of reader.read({ signal: tc.signal })) { + expect(batch[0]!.payload.byteLength).toBe(400) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(credits(next)).toEqual([600n, 400n, 600n, 400n]) +}) + +test('confirms a pending commit while retained messages withhold new stream read credit', async (tc) => { + let fake = makeFakeTopicDriver() + let starts = 0 + using reader = createTopicReader(fake.driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 1000n, + onPartitionSessionStart: async () => { + starts += 1 + }, + }) + let first = await initStream(fake) + first.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await first.waitForStartResponse() + first.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: 1000n, + messages: [0n, 1n].map((offset) => ({ + offset, + seqNo: offset + 1n, + data: new Uint8Array([1]), + })), + }) + ) + let pending: Promise | undefined + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + pending = reader.commit(batch) + break + } + await first.waitForCommit() + first.disconnect() + let next = await fake.waitForNextStream() + await next.waitForInit() + next.respond(initResponse('')) + next.respond(startPartitionSession({ partitionSessionId: 2n, partitionId: 0n })) + await next.waitForStartResponse() + let committed = await next.waitForCommit() + expect(committed.commitOffsets).toMatchObject([ + { partitionSessionId: 2n, offsets: [{ start: 0n, end: 1n }] }, + ]) + next.respond(commitOffsetResponse([{ partitionSessionId: 2n, committedOffset: 1n }])) + await pending + expect(starts).toBe(2) + expect(reader.bufferedBytes).toBe(1000n) + expect(credits(next)).toEqual([]) +}) diff --git a/packages/topic/src/reader/reader.lifecycle.test.ts b/packages/topic/src/reader/reader.lifecycle.test.ts index 2788ea8f..5d3e4a3f 100644 --- a/packages/topic/src/reader/reader.lifecycle.test.ts +++ b/packages/topic/src/reader/reader.lifecycle.test.ts @@ -1,9 +1,10 @@ import { getEventListeners } from 'node:events' -import { expect, test } from 'vitest' +import { expect, test, vi } from 'vitest' import { createTopicReader } from './index.ts' import { initResponse, makeFakeTopicDriver, settle } from './reader.fixtures.ts' +import { ReaderTransport } from './transport.ts' // Resource-lifecycle coverage for the reader against the fake wire, mirroring the // writer's leak tests in writer.contract.test.ts: create/destroy churn must stay flat @@ -11,6 +12,59 @@ import { initResponse, makeFakeTopicDriver, settle } from './reader.fixtures.ts' // signal must not accumulate listeners across lifecycles. The suite runs with // --expose-gc (see vitest.config.ts), so globalThis.gc is available. +// A server failure is a protocol event; only an injected effect exception can +// deterministically exercise the runtime's fault path without a terminal transition. +test('closes the stream and clears timers when a reader effect throws', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c' }) + let stream = await waitForNextStream() + await stream.waitForInit() + let error = new Error('Injected transport send failure') + using send = vi.spyOn(ReaderTransport.prototype, 'send').mockImplementation(() => { + throw error + }) + stream.respond(initResponse()) + await settle() + expect(stream.wasAborted()).toBe(true) + expect(vi.getTimerCount()).toBe(0) + await expect(reader.close()).rejects.toBe(error) + await vi.advanceTimersByTimeAsync(60_000) + expect(send).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } +}) + +test('preserves a zero destroy reason for reads commits and close', async () => { + let { driver } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c' }) + reader.destroy(0) + let iterator = reader.read()[Symbol.asyncIterator]() + await expect(iterator.next()).rejects.toBe(0) + await expect(reader.commit([])).rejects.toBe(0) + await expect(reader.close()).rejects.toBe(0) +}) + +test('releases the live transport and every timer before graceful close resolves', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c' }) + let stream = await waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse()) + await stream.waitForReadRequest() + expect(vi.getTimerCount()).toBeGreaterThan(0) + await reader.close() + expect(stream.wasAborted()).toBe(true) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } +}) + // Bring one reader to `ready`, then destroy it. Returns nothing so the // reader/driver/stream locals leave the stack and become collectable. let spinUpAndDestroy = async function spinUpAndDestroy(): Promise { diff --git a/packages/topic/src/reader/reader.memory.test.ts b/packages/topic/src/reader/reader.memory.test.ts new file mode 100644 index 00000000..4e958a6b --- /dev/null +++ b/packages/topic/src/reader/reader.memory.test.ts @@ -0,0 +1,400 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { gzipSync } from 'node:zlib' + +import { create, toBinary } from '@bufbuild/protobuf' +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { + Codec, + StreamReadMessage_FromServerSchema, + StreamReadMessage_ReadResponseSchema, +} from '@ydbjs/api/topic' +import { expect, test, vi } from 'vitest' + +import { GZIP_CODEC } from '../codec.ts' +import type { TopicPartitionSession } from '../partition-session.ts' +import { TopicReader } from './reader.ts' +import { ReaderTransport } from './transport.ts' +import { + failureResponse, + initResponse, + makeFakeTopicDriver, + makeFakeTx, + readResponse, + settle, + startPartitionSession, + stopPartitionSession, +} from './reader.fixtures.ts' + +// FinalizationRegistry observes reclamation without deref() keeping a target alive +// in the GC job. The test retains the reader, so dropping that object cannot pass it. +let collectGarbage = async function collectGarbage(): Promise { + for (let i = 0; i < 20; i++) { + // oxlint-disable-next-line no-await-in-loop + await delay(1) + globalThis.gc!() + // oxlint-disable-next-line no-await-in-loop + await delay(1) + } +} + +let memoryProbe = function memoryProbe( + registry: FinalizationRegistry, + calls: string[], + id: string +) { + let retained = new Uint8Array(32 * 1024) + registry.register(retained, id) + return () => calls.push(`${id}:${retained.byteLength}`) +} + +let startHook = function startHook(registry: FinalizationRegistry, probe: () => unknown) { + let grants = 0 + return async (session: TopicPartitionSession) => { + probe() + registry.register(session, `session-${grants++}`) + } +} + +let stopHook = function stopHook(probe: () => unknown) { + return async () => { + probe() + } +} + +let retainedCodec = function retainedCodec(probe: () => unknown) { + return { + codec: Codec.RAW, + compress: (data: Uint8Array) => data, + decompress: (data: Uint8Array) => { + probe() + return data + }, + } +} + +let bufferedReader = async function bufferedReader( + count = 2, + tx?: ReturnType['tx'] +) { + let reclaimed = new Set() + let registry = new FinalizationRegistry((id) => reclaimed.add(id)) + let calls: string[] = [] + let fake = makeFakeTopicDriver() + let reader = new TopicReader( + fake.driver, + { + topic: '/t', + consumer: 'c', + codecMap: new Map([[Codec.RAW, retainedCodec(memoryProbe(registry, calls, 'codec'))]]), + onPartitionSessionStart: startHook( + registry, + memoryProbe(registry, calls, 'start-hook') + ), + onPartitionSessionStop: stopHook(memoryProbe(registry, calls, 'stop-hook')), + onCommittedOffset: memoryProbe(registry, calls, 'commit-hook'), + }, + tx ? { tx } : undefined + ) + let stream = await fake.waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse()) + await stream.waitForReadRequest() + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + if (count > 0) { + stream.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: BigInt(count * 256 * 1024), + messages: Array.from({ length: count }, (_, i) => { + let data = new Uint8Array(256 * 1024) + registry.register(data, `payload-${i}`) + return { offset: BigInt(i), seqNo: BigInt(i + 1), data } + }), + }) + ) + } + await settle() + return { reader, stream, fake, reclaimed, registry, calls } +} + +let readOne = async function readOne(reader: TopicReader, signal: AbortSignal): Promise { + for await (let batch of reader.read({ limit: 1, signal })) { + expect(batch).toHaveLength(1) + break + } +} + +let readTail = async function readTail( + reader: TopicReader, + signal: AbortSignal +): Promise { + let offsets: bigint[] = [] + for await (let batch of reader.read({ signal })) { + offsets.push(...batch.map((message) => message.offset!)) + } + return offsets +} + +// Server packet boundaries and compression ratios must be fixed to distinguish +// encoded read credit from decoded retention independently of YDB batching. +test('decodes only requested messages from a compressed response', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let decodedBytes = 0 + using reader = new TopicReader(driver, { + topic: '/t', + consumer: 'c', + maxBufferBytes: 65536n, + codecMap: new Map([ + [ + Codec.GZIP, + { + ...GZIP_CODEC, + decompress(data) { + let decoded = GZIP_CODEC.decompress(data) + decodedBytes += decoded.byteLength + return decoded + }, + }, + ], + ]), + }) + let stream = await waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse()) + await stream.waitForReadRequest() + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 0n })) + await stream.waitForStartResponse() + let data = gzipSync(new Uint8Array(256 * 1024)) + let response = create( + StreamReadMessage_FromServerSchema, + readResponse({ + partitionSessionId: 1n, + codec: Codec.GZIP, + bytesSize: 0n, + messages: Array.from({ length: 64 }, (_, i) => ({ + offset: BigInt(i), + seqNo: BigInt(i + 1), + data, + })), + }) + ) + if (response.serverMessage.case !== 'readResponse') { + throw new Error('Expected read response') + } + for (let partition of response.serverMessage.value.partitionData) { + for (let batch of partition.batches) { + for (let message of batch.messageData) message.uncompressedSize = 262144n + } + } + let bytesSize = BigInt( + toBinary(StreamReadMessage_ReadResponseSchema, response.serverMessage.value).length + ) + response.serverMessage.value.bytesSize = bytesSize + expect(bytesSize).toBeLessThan(65536n) + stream.respond(response) + await settle() + expect(decodedBytes).toBe(0) + await readOne(reader, tc.signal) + expect(decodedBytes).toBe(262144) + expect(reader.bufferedBytes).toBe(bytesSize) + expect(stream.sent.filter((frame) => frame.clientMessage.case === 'readRequest')).toHaveLength( + 1 + ) +}) + +test('releases unread payloads and callbacks after destroy while the reader is retained', async () => { + let fixture = await bufferedReader() + using reader = fixture.reader + reader.destroy(new Error('Hard stop')) + await settle() + await collectGarbage() + expect(fixture.reclaimed).toEqual( + new Set([ + 'payload-0', + 'payload-1', + 'session-0', + 'start-hook', + 'stop-hook', + 'commit-hook', + 'codec', + ]) + ) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('releases unread payloads after a fatal server status', async () => { + let fixture = await bufferedReader() + using reader = fixture.reader + fixture.stream.respond(failureResponse(StatusIds_StatusCode.UNAUTHORIZED)) + await settle() + await expect(reader.read()[Symbol.asyncIterator]().next()).rejects.toBeDefined() + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(true) + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('reclaims an active partition after an internal commit effect fails', async (tc) => { + let fixture = await bufferedReader(1) + using reader = fixture.reader + let reason = new Error('Injected commit send failure') + for await (let batch of reader.read({ signal: tc.signal })) { + using send = vi.spyOn(ReaderTransport.prototype, 'send').mockImplementation(() => { + throw reason + }) + await expect(reader.commit(batch)).rejects.toBe(reason) + expect(send).toHaveBeenCalledTimes(1) + break + } + await settle() + await collectGarbage() + expect(fixture.reclaimed.has('session-0')).toBe(true) + expect(fixture.stream.wasAborted()).toBe(true) + await expect(reader.close()).rejects.toBe(reason) +}) + +test('releases unread carry after a split read is destroyed', async (tc) => { + let fixture = await bufferedReader() + using reader = fixture.reader + await readOne(reader, tc.signal) + reader.destroy(new Error('Hard stop')) + await settle() + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(true) + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('releases unread messages while a destroyed iterator remains suspended at yield', async (tc) => { + let fixture = await bufferedReader() + using reader = fixture.reader + let iterator = reader.read({ limit: 1, signal: tc.signal })[Symbol.asyncIterator]() + let first = await iterator.next() + try { + reader.destroy(new Error('Hard stop')) + await settle() + await collectGarbage() + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(first.value?.[0]?.payload.byteLength).toBe(256 * 1024) + expect(reader.bufferedBytes).toBe(0n) + } finally { + await iterator.return?.() + } +}) + +test('does not restore a buffered read window after a hard stop', async () => { + let fixture = await bufferedReader() + using reader = fixture.reader + let reason = new Error('Hard stop') + let iterator = reader.read({ batchWindowMs: 60_000 })[Symbol.asyncIterator]() + let next = iterator.next().catch((error: unknown) => error) + await settle() + reader.destroy(reason) + expect(await next).toBe(reason) + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(true) + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('rejects a queued read result when destroy wins before its continuation', async () => { + let fixture = await bufferedReader() + using reader = fixture.reader + let reason = new Error('Hard stop') + let iterator = reader.read()[Symbol.asyncIterator]() + let next = iterator.next().catch((error: unknown) => error) + reader.destroy(reason) + expect(await next).toBe(reason) + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(true) + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('keeps a cleanly closed tail readable across GC and releases it after reading', async (tc) => { + let fixture = await bufferedReader() + using reader = fixture.reader + await reader.close() + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(false) + expect(fixture.reclaimed.has('codec')).toBe(false) + expect(fixture.reclaimed.has('start-hook')).toBe(true) + expect(fixture.reclaimed.has('stop-hook')).toBe(true) + expect(fixture.reclaimed.has('commit-hook')).toBe(true) + expect(await readTail(reader, tc.signal)).toEqual([0n, 1n]) + await collectGarbage() + expect(fixture.reclaimed.has('payload-0')).toBe(true) + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(fixture.reclaimed.has('codec')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('clears transaction offsets on rollback after the reader was cleanly closed', async (tc) => { + let tx = makeFakeTx() + let fixture = await bufferedReader(1, tx.tx) + using reader = fixture.reader + await readOne(reader, tc.signal) + await reader.close() + await tx.rollback(new Error('Rollback')) + await tx.close(false) + await collectGarbage() + expect(fixture.reclaimed.has('session-0')).toBe(true) + expect(fixture.reclaimed.has('start-hook')).toBe(true) + expect(fixture.reclaimed.has('codec')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('clears transaction offsets after successful transaction finalization', async (tc) => { + let tx = makeFakeTx() + let fixture = await bufferedReader(1, tx.tx) + using reader = fixture.reader + await readOne(reader, tc.signal) + await tx.commit() + await tx.close(true) + await collectGarbage() + expect(fixture.reclaimed.has('session-0')).toBe(true) + expect(fixture.reclaimed.has('start-hook')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('preserves delivered transaction offsets after a clean close followed by disposal', async (tc) => { + let tx = makeFakeTx() + let fixture = await bufferedReader(2, tx.tx) + using reader = fixture.reader + await readOne(reader, tc.signal) + await reader.close() + reader[Symbol.dispose]() + await expect(reader.close()).resolves.toBeUndefined() + await tx.commit() + expect(fixture.fake.txOffsetRequests).toHaveLength(1) + expect(fixture.fake.txOffsetRequests[0]!.topics[0]!.partitions[0]!.partitionOffsets).toEqual([ + expect.objectContaining({ start: 0n, end: 1n }), + ]) + await tx.close(true) + await collectGarbage() + expect(fixture.reclaimed.has('payload-1')).toBe(true) + expect(fixture.reclaimed.has('session-0')).toBe(true) + expect(reader.bufferedBytes).toBe(0n) +}) + +test('reclaims stopped partitions without pending commits while the reader stays live', async () => { + let fixture = await bufferedReader(0) + using reader = fixture.reader + fixture.stream.respond(stopPartitionSession({ partitionSessionId: 1n })) + for (let i = 1; i < 32; i++) { + fixture.stream.respond( + startPartitionSession({ partitionSessionId: BigInt(i + 1), partitionId: BigInt(i) }) + ) + // oxlint-disable-next-line no-await-in-loop + await settle() + fixture.stream.respond(stopPartitionSession({ partitionSessionId: BigInt(i + 1) })) + // oxlint-disable-next-line no-await-in-loop + await settle() + } + await collectGarbage() + expect( + Array.from(fixture.reclaimed).filter((id) => id.startsWith('session-')).length + ).toBeGreaterThanOrEqual(31) + expect(reader.bufferedBytes).toBe(0n) +}) diff --git a/packages/topic/src/reader/reader.model.test.ts b/packages/topic/src/reader/reader.model.test.ts index 220df402..aabb68ea 100644 --- a/packages/topic/src/reader/reader.model.test.ts +++ b/packages/topic/src/reader/reader.model.test.ts @@ -2,6 +2,8 @@ import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { YDBError } from '@ydbjs/error' import { expect, test } from 'vitest' +import type { TopicPartitionSession } from '../partition-session.ts' + import { type OffsetRange, type ReaderCtx, @@ -10,21 +12,19 @@ import { type ReaderOutput, type ReaderState, createReaderCtx, - mergeRanges, partitionKey, readerTransition, } from './reader-state.ts' -import { - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.ts' - -// Model-based / property test. It wires the two REAL pure transitions -// (readerTransition + transportTransition) to a protocol-faithful server model and -// drives them with random sequences of consumer calls, commits, network events, +type TransportEvent = + | { type: 'transport.connect' } + | { type: 'transport.destroy'; reason?: unknown } + | { type: 'transport.init'; sessionId: string } + | { type: 'transport.message'; message: unknown } + | { type: 'transport.ended' } + | { type: 'transport.error'; error: unknown } + +// Model-based / property test. The reader transition runs against an independent +// connection and server model, driven by consumer calls, commits, network events, // server assignments/deliveries (with server-side offset holes)/acks, // server-initiated partition stops (graceful AND force) and timer firings — // including the per-partition partition_graceful_timeout and partition_reassign_gc @@ -84,14 +84,6 @@ let modelMerge = function modelMerge(ranges: OffsetRange[]): OffsetRange[] { return merged } -let rangesLength = function rangesLength(ranges: OffsetRange[]): bigint { - let total = 0n - for (let range of ranges) { - total += range.end - range.start - } - return total -} - let clampRanges = function clampRanges(ranges: OffsetRange[], floor: bigint): OffsetRange[] { let clamped: OffsetRange[] = [] for (let range of ranges) { @@ -167,7 +159,6 @@ type Waiter = { type Sim = { readerState: ReaderState readerCtx: ReaderCtx - transportState: TransportState readerEvents: ReaderEvent[] transportEvents: TransportEvent[] @@ -193,8 +184,8 @@ type Sim = { // The async onPartitionSessionStop hook: the stop_hook effect queues here and a // randomly interleaved action completes it as reader.partition.stop_ready. - pendingStopHooks: { partitionKey: string; grantId: number }[] - stopReadyDone: Set // `${partitionKey}#${grantId}` — hook completed for that grant + pendingStopHooks: TopicPartitionSession[] + stopReadyDone: Set // hook completed for that grant gracefulTimeoutFired: Set // partitionKey — the escalation legalizing an early stop_response // Session ids are globally unique here (nextSessionId is monotonic), so both sets @@ -226,7 +217,6 @@ let mkSim = function mkSim( return { readerState: 'idle', readerCtx: createReaderCtx({ maxBufferBytes }), - transportState: 'idle', readerEvents: [], transportEvents: [], armed: new Set(), @@ -508,8 +498,8 @@ let applyReaderEffect = function applyReaderEffect(sim: Sim, effect: ReaderEffec let entry = sim.readerCtx.partitions.get(part.key) let handshakeDone = entry !== undefined && - sim.stopReadyDone.has(`${part.key}#${entry.grantId}`) && - entry.pendingCommits.length === 0 + sim.stopReadyDone.has(entry.session) && + entry.commitWaiters.length === 0 if (!handshakeDone && !sim.gracefulTimeoutFired.has(part.key)) { throw new Error( `stop_response for ${part.key} before stop_ready and drained commits` @@ -534,19 +524,14 @@ let applyReaderEffect = function applyReaderEffect(sim: Sim, effect: ReaderEffec // start_response and re-sends reconciled commits. sim.readerEvents.push({ type: 'reader.partition.start_ready', - partitionSessionId: effect.partitionSessionId, - partitionKey: effect.partitionKey, - grantId: effect.grantId, + session: effect.session, }) break } case 'reader.effect.partition.stop_hook': { // The graceful-stop hook runs detached in the runtime — the model completes // it as a separate, randomly interleaved action (or in the cooldown flush). - sim.pendingStopHooks.push({ - partitionKey: effect.partitionKey, - grantId: effect.grantId, - }) + sim.pendingStopHooks.push(effect.session) break } case 'reader.effect.send.start_response': { @@ -561,9 +546,6 @@ let applyReaderEffect = function applyReaderEffect(sim: Sim, effect: ReaderEffec } case 'reader.effect.send.update_token': break - case 'reader.effect.transport.close': - sim.transportEvents.push({ type: 'transport.close' }) - break case 'reader.effect.timer.schedule': { let key = 'partitionKey' in effect ? `${effect.which}:${effect.partitionKey}` : effect.which @@ -586,31 +568,6 @@ let applyReaderEffect = function applyReaderEffect(sim: Sim, effect: ReaderEffec } } -let applyTransportEffect = function applyTransportEffect(sim: Sim, effect: TransportEffect): void { - switch (effect.type) { - case 'transport.effect.open_stream': - sim.streamOpen = true - sim.initPending = true - sim.credit = 0n - // Ephemeral assignments — and the sparse per-session commit state — die - // with the old stream; only the durable watermark survives. - for (let part of sim.partitions) { - part.partitionSessionId = undefined - part.ready = false - part.stopping = false - part.deliveredUpTo = part.durableCommitted - part.sessionRanges = [] - part.unackedCommits = 0 - } - break - case 'transport.effect.close_stream': - case 'transport.effect.finalize': - sim.streamOpen = false - sim.initPending = false - break - } -} - let processReaderEvent = function processReaderEvent(sim: Sim, event: ReaderEvent): void { let runtime = { state: sim.readerState, @@ -627,32 +584,57 @@ let processReaderEvent = function processReaderEvent(sim: Sim, event: ReaderEven } } +// Stream replacement drops ephemeral assignments and sparse commit coverage; +// only the server's durable watermark survives. Production transport is tested separately. let processTransportEvent = function processTransportEvent(sim: Sim, event: TransportEvent): void { - let runtime = { - state: sim.transportState, - signal: NEVER, - emit: (output: TransportOutput) => { - let mapped: ReaderEvent | null = - output.type === 'transport.stream.init_response' - ? { type: 'reader.stream.init_response', sessionId: output.sessionId } - : output.type === 'transport.stream.disconnected' - ? { - type: 'reader.stream.disconnected', - ...('error' in output ? { error: output.error } : {}), - } - : classify(output.message) - if (mapped) { - sim.readerEvents.push(mapped) + switch (event.type) { + case 'transport.connect': + if (sim.terminal) { + return } - }, - dispatch: (next: TransportEvent) => sim.transportEvents.push(next), + sim.streamOpen = true + sim.initPending = true + sim.credit = 0n + for (let part of sim.partitions) { + part.partitionSessionId = undefined + part.ready = false + part.stopping = false + part.deliveredUpTo = part.durableCommitted + part.sessionRanges = [] + part.unackedCommits = 0 + } + return + case 'transport.destroy': + sim.streamOpen = false + sim.initPending = false + return } - let result = transportTransition({}, event, runtime) - if (result?.state) { - sim.transportState = result.state + if (!sim.streamOpen || sim.terminal) { + return } - for (let effect of result?.effects ?? []) { - applyTransportEffect(sim, effect) + switch (event.type) { + case 'transport.init': + sim.initPending = false + sim.readerEvents.push({ + type: 'reader.stream.init_response', + sessionId: event.sessionId, + }) + return + case 'transport.message': { + let mapped = classify(event.message) + if (mapped) { + sim.readerEvents.push(mapped) + } + return + } + case 'transport.ended': + case 'transport.error': + sim.streamOpen = false + sim.initPending = false + sim.readerEvents.push({ + type: 'reader.stream.disconnected', + ...('error' in event ? { error: event.error } : {}), + }) } } @@ -787,7 +769,7 @@ let issueCommit = function issueCommit(sim: Sim, key: string, selected: MirrorMe for (let message of selected) { message.requested = true } - let ranges = mergeRanges(selected.map((m) => ({ start: m.start, end: m.end }))) + let ranges = modelMerge(selected.map((m) => ({ start: m.start, end: m.end }))) sim.requestedUnion.set(key, modelMerge([...(sim.requestedUnion.get(key) ?? []), ...ranges])) let waiterId = sim.nextWaiterId++ sim.waiters.set(waiterId, { @@ -817,6 +799,9 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { let ctx = sim.readerCtx if (sim.terminal) { + if (sim.armed.size !== 0 || sim.streamOpen) { + throw new Error(`${where}: terminal but timers or transport remain live`) + } if (ctx.partitions.size !== 0 || ctx.sessionIndex.size !== 0) { throw new Error(`${where}: terminal but ctx not cleared`) } @@ -839,8 +824,26 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { } } - if (ctx.inFlightBytes < 0n) { - throw new Error(`${where}: negative flow-control inFlight=${ctx.inFlightBytes}`) + let retainedBytes = sim.unreleased.reduce((total, bytes) => total + bytes, 0n) + if (ctx.bufferedBytes !== retainedBytes) { + throw new Error( + `${where}: retained flow-control bytes ${ctx.bufferedBytes} differ from queued ${retainedBytes}` + ) + } + + if (sim.readerState === 'ready') { + let expectedCredit = ctx.limits.maxBufferBytes - retainedBytes + if (sim.credit !== expectedCredit) { + throw new Error(`${where}: expected server credit ${expectedCredit}, got ${sim.credit}`) + } + } + + if (sim.readerState === 'waiting-credit') { + if (sim.credit !== 0n || retainedBytes < ctx.limits.maxBufferBytes) { + throw new Error( + `${where}: initialized session waits for credit without a full retained buffer` + ) + } } // sessionIndex is consistent: every entry points to a partition whose current @@ -852,9 +855,9 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { `${where}: sessionIndex ${partitionSessionId} -> missing partition ${key}` ) } - if (entry.partitionSessionId !== partitionSessionId) { + if (entry.session.partitionSessionId !== partitionSessionId) { throw new Error( - `${where}: stale sessionIndex ${partitionSessionId} -> ${key} (current ${entry.partitionSessionId})` + `${where}: stale sessionIndex ${partitionSessionId} -> ${key} (current ${entry.session.partitionSessionId})` ) } } @@ -863,25 +866,20 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { if (entry.deliveredWatermark < entry.partitionCommittedOffset) { throw new Error(`${where}: delivery watermark below committed on ${key}`) } - // Wire ranges are normalized and pairwise disjoint. A fully claimed duplicate - // legitimately has no wire ranges while its target waits for the same watermark. - let all: OffsetRange[] = [] - for (let pending of entry.pendingCommits) { + for (let pending of entry.commitWaiters) { if (pending.targetOffset <= entry.partitionCommittedOffset) { throw new Error(`${where}: confirmed pending target on ${key}`) } - assertNormalized(pending.wireRanges, `${where}: pending commit on ${key}`) - all.push(...pending.wireRanges) - } - let union = modelMerge(all) - if (rangesLength(union) !== rangesLength(all)) { - throw new Error(`${where}: overlapping pending commits on ${key}`) } - // Claimed coverage is exactly the pending remainders above the watermark — - // the guard recordCommit subtracts to never re-send a claimed offset. - let expected = clampRanges(union, entry.partitionCommittedOffset) - if (!sameRanges(entry.claimedRanges, expected)) { - throw new Error(`${where}: claimedRanges drifted from pending coverage on ${key}`) + // Compare against the consumer's original requests, independently of how the + // implementation batches their coverage or deduplicates concurrent callers. + let requests = Array.from(sim.waiters.values()) + .filter((waiter) => waiter.partitionKey === key && waiter.state === 'pending') + .flatMap((waiter) => waiter.ranges) + let expected = clampRanges(modelMerge(requests), entry.partitionCommittedOffset) + assertNormalized(entry.pendingRanges, `${where}: pending ranges on ${key}`) + if (!sameRanges(entry.pendingRanges, expected)) { + throw new Error(`${where}: pending ranges differ from consumer requests on ${key}`) } // The model's independently tracked delivery watermark agrees with the FSM's. let mirror = sim.mirrors.get(key) @@ -937,6 +935,7 @@ let runOne = function runOne( let live = sim.readerState === 'connecting' || sim.readerState === 'ready' || + sim.readerState === 'waiting-credit' || sim.readerState === 'reconnecting' let actions: Array<{ w: number; run: () => void }> = [] @@ -1018,11 +1017,10 @@ let runOne = function runOne( w: 4, run: () => { let hook = sim.pendingStopHooks.splice(i, 1)[0]! - sim.stopReadyDone.add(`${hook.partitionKey}#${hook.grantId}`) + sim.stopReadyDone.add(hook) sim.readerEvents.push({ type: 'reader.partition.stop_ready', - partitionKey: hook.partitionKey, - grantId: hook.grantId, + session: hook, }) }, }) @@ -1207,9 +1205,6 @@ let runOne = function runOne( continue } - // The consumer keeps up and the server grants effectively unlimited credit — - // the cooldown drives the system to its fixed point, not the flow-control edge. - sim.credit = 1_000_000n while (sim.unreleased.length > 0) { sim.readerEvents.push({ type: 'reader.read_release', bytes: sim.unreleased.shift()! }) } @@ -1226,6 +1221,7 @@ let runOne = function runOne( part.partitionSessionId !== undefined && part.ready && !part.stopping && + sim.credit > 0n && part.deliveredUpTo < part.availableUpTo ) { deliver(sim, part, randInt) @@ -1235,11 +1231,10 @@ let runOne = function runOne( runToQuiescence(sim) while (sim.pendingStopHooks.length > 0) { let hook = sim.pendingStopHooks.shift()! - sim.stopReadyDone.add(`${hook.partitionKey}#${hook.grantId}`) + sim.stopReadyDone.add(hook) sim.readerEvents.push({ type: 'reader.partition.stop_ready', - partitionKey: hook.partitionKey, - grantId: hook.grantId, + session: hook, }) progressed = true } diff --git a/packages/topic/src/reader/reader.partition-lifecycle.test.ts b/packages/topic/src/reader/reader.partition-lifecycle.test.ts index 34ae7165..1497b0ff 100644 --- a/packages/topic/src/reader/reader.partition-lifecycle.test.ts +++ b/packages/topic/src/reader/reader.partition-lifecycle.test.ts @@ -75,6 +75,47 @@ let stopResponses = function stopResponses(sent: SentFrames): number { return sent.filter((m) => m.clientMessage.case === 'stopPartitionSessionResponse').length } +test('keeps registered callbacks after the caller mutates its options', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let calls: string[] = [] + let options = { + topic: '/t', + consumer: 'c', + onCommittedOffset: () => { + calls.push('registered-commit') + }, + onPartitionSessionStop: async () => { + calls.push('registered-stop') + }, + } + using reader = createTopicReader(driver, options) + options.onCommittedOffset = () => { + calls.push('replacement-commit') + } + options.onPartitionSessionStop = async () => { + calls.push('replacement-stop') + } + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('a') }], + }) + ) + let messages = await collect(reader, 1, tc.signal) + let committing = reader.commit(messages) + await stream.waitForCommit() + stream.respond(commitOffsetResponse([{ partitionSessionId: 1n, committedOffset: 1n }])) + await committing + stream.respond( + stopPartitionSession({ partitionSessionId: 1n, graceful: false, committedOffset: 1n }) + ) + await settle() + expect(calls).toEqual(['registered-commit', 'registered-stop']) +}) + // The protocol's soft stop holds the partition on the server until the client sends // StopPartitionSessionResponse — the delay is the mechanism that lets the app finish // processing and commit. onPartitionSessionStop is that last-chance window: it runs @@ -205,7 +246,9 @@ test('escalates a stalled graceful stop to force without answering either stop', // the callback they are distinguishable via session.isStopped (false during the // window, true once lost). expect(stopCalls).toBe(2) - expect(stopped.payloads).toEqual([expect.objectContaining({ partitionId: 10n, reason: 'lost' })]) + expect(stopped.payloads).toEqual([ + expect.objectContaining({ partitionId: 10n, reason: 'lost' }), + ]) // The uncovered commit is held for a possible re-grant, not settled by the stop. expect(heldSettled).toBe(false) diff --git a/packages/topic/src/reader/reader.read-loop.test.ts b/packages/topic/src/reader/reader.read-loop.test.ts index b811d747..52cff9a7 100644 --- a/packages/topic/src/reader/reader.read-loop.test.ts +++ b/packages/topic/src/reader/reader.read-loop.test.ts @@ -7,6 +7,7 @@ import { readResponse, settle, startPartitionSession, + stopPartitionSession, } from './reader.fixtures.ts' // read()-loop semantics against a fake streamRead: limit validation, abort @@ -201,10 +202,154 @@ test('redelivers messages dequeued by an aborted read()', async (tc) => { expect(redelivered).toEqual([0n, 1n, 2n, 3n, 4n]) // Both response boundaries completed in the yielded batch, so their exact combined // server-accounted size is returned only now. + await settle() expect(readRequests(stream.sent)).toEqual([1000n, 500n]) expect(reader.bufferedBytes).toBe(0n) }) +// Live server response boundaries and rebalance timing cannot deterministically +// place a stop between two yields from the same response. +test('preserves the remaining slices when a read is aborted between yields', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c', maxBufferBytes: 1000n }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: 300n, + messages: [0n, 1n, 2n].map((offset) => ({ + offset, + seqNo: offset + 1n, + data: bytes(String(offset)), + })), + }) + ) + let ac = new AbortController() + let reason = new Error('stop reading') + let iterator = reader.read({ limit: 1, signal: ac.signal })[Symbol.asyncIterator]() + expect((await iterator.next()).value?.map((message) => message.offset)).toEqual([0n]) + ac.abort(reason) + await expect(iterator.next()).rejects.toBe(reason) + expect(reader.bufferedBytes).toBe(300n) + expect(readRequests(stream.sent)).toEqual([1000n]) + let cancelled = reader.read({ limit: 1, signal: ac.signal })[Symbol.asyncIterator]() + await expect(cancelled.next()).rejects.toBe(reason) + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([1n]) + break + } + expect(reader.bufferedBytes).toBe(300n) + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch.map((message) => message.offset)).toEqual([2n]) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(readRequests(stream.sent)).toEqual([1000n, 300n]) +}) + +test('drops revoked slices and releases their response credit once between yields', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c', maxBufferBytes: 1000n }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + stream.respond(startPartitionSession({ partitionSessionId: 2n, partitionId: 11n })) + await settle() + stream.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: 300n, + messages: [0n, 1n, 2n].map((offset) => ({ + offset, + seqNo: offset + 1n, + data: bytes(String(offset)), + })), + }) + ) + let iterable = reader.read({ limit: 1, batchWindowMs: 5, signal: tc.signal }) + let iterator = iterable[Symbol.asyncIterator]() + try { + expect((await iterator.next()).value?.map((message) => message.offset)).toEqual([0n]) + stream.respond(stopPartitionSession({ partitionSessionId: 1n })) + await settle() + expect((await iterator.next()).value).toEqual([]) + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(readRequests(stream.sent)).toEqual([1000n, 300n]) + stream.respond( + readResponse({ + partitionSessionId: 2n, + bytesSize: 100n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('live') }], + }) + ) + expect((await iterator.next()).value?.map((message) => text(message.payload))).toEqual([ + 'live', + ]) + } finally { + await iterator.return?.() + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(readRequests(stream.sent)).toEqual([1000n, 300n, 100n]) +}) + +test('retains shared response credit until its live partition finishes after a stop', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c', maxBufferBytes: 1000n }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + stream.respond(startPartitionSession({ partitionSessionId: 2n, partitionId: 11n })) + await settle() + let response = readResponse({ + partitionSessionId: 1n, + bytesSize: 400n, + messages: [0n, 1n].map((offset) => ({ + offset, + seqNo: offset + 1n, + data: bytes('stopped'), + })), + }) + let live = readResponse({ + partitionSessionId: 2n, + messages: [0n, 1n].map((offset) => ({ offset, seqNo: offset + 1n, data: bytes('live') })), + }) + if ( + response.serverMessage.case !== 'readResponse' || + live.serverMessage.case !== 'readResponse' + ) { + throw new Error('Expected read response fixtures') + } + response.serverMessage.value.partitionData.push(live.serverMessage.value.partitionData[0]!) + stream.respond(response) + let iterator = reader.read({ limit: 1, signal: tc.signal })[Symbol.asyncIterator]() + try { + expect((await iterator.next()).value?.map((message) => text(message.payload))).toEqual([ + 'stopped', + ]) + stream.respond(stopPartitionSession({ partitionSessionId: 1n })) + await settle() + expect((await iterator.next()).value?.map((message) => text(message.payload))).toEqual([ + 'live', + ]) + expect(reader.bufferedBytes).toBe(400n) + expect(readRequests(stream.sent)).toEqual([1000n]) + } finally { + await iterator.return?.() + } + for await (let batch of reader.read({ limit: 1, signal: tc.signal })) { + expect(batch.map((message) => [text(message.payload), message.offset])).toEqual([ + ['live', 1n], + ]) + break + } + await settle() + expect(reader.bufferedBytes).toBe(0n) + expect(readRequests(stream.sent)).toEqual([1000n, 400n]) +}) + test('accepts an oversized response and replenishes the full server-reported bytes', async (tc) => { let { driver, waitForNextStream } = makeFakeTopicDriver() using reader = createTopicReader(driver, { @@ -254,3 +399,43 @@ test('accepts an oversized response and replenishes the full server-reported byt await settle() expect(readRequests(stream.sent)).toEqual([1024n, 5000n, 600n]) }) + +test('returns the server response size once after all 100 message batches are delivered', async (tc) => { + // The fake fixes response framing and bytesSize independently of payload lengths. + let { driver, waitForNextStream } = makeFakeTopicDriver() + let maxBufferBytes = 8n * 1024n * 1024n + let responseBytes = 10n * 1024n * 1024n + using reader = createTopicReader(driver, { topic: '/t', consumer: 'c', maxBufferBytes }) + let stream = await primeStream(reader, waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + bytesSize: responseBytes, + messages: Array.from({ length: 10_000 }, (_, offset) => ({ + offset: BigInt(offset), + seqNo: BigInt(offset + 1), + data: new Uint8Array([offset % 256]), + })), + }) + ) + await settle() + expect(reader.bufferedBytes).toBe(responseBytes) + + let deliveredBatches = 0 + for await (let batch of reader.read({ limit: 100, signal: tc.signal })) { + expect(batch).toHaveLength(100) + deliveredBatches += 1 + await settle() + let lastBatch = deliveredBatches === 100 + expect(reader.bufferedBytes).toBe(lastBatch ? 0n : responseBytes) + expect(readRequests(stream.sent)).toEqual( + lastBatch ? [maxBufferBytes, responseBytes] : [maxBufferBytes] + ) + if (lastBatch) { + break + } + } + expect(deliveredBatches).toBe(100) +}) diff --git a/packages/topic/src/reader/reader.ts b/packages/topic/src/reader/reader.ts index bb655bf0..6090a27f 100644 --- a/packages/topic/src/reader/reader.ts +++ b/packages/topic/src/reader/reader.ts @@ -1,3 +1,5 @@ +import { isPromise } from 'node:util/types' + import { create } from '@bufbuild/protobuf' import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { @@ -45,21 +47,23 @@ import { type ReaderRuntime, createReaderRuntime, } from './reader-runtime.js' -import { type OffsetRange, partitionKey } from './reader-state.js' +import { type OffsetRange, type ReaderMessage, partitionKey } from './reader-state.js' import type { TopicReadOptions, TopicReaderOptions, TopicTxReader } from './types.js' let dbg = loggers.topic.extend('reader') -// A decoded ReadResponse: the consumer takes the messages, then the reader releases -// the response's flow-control credit (backpressure — credit is granted only as the -// consumer keeps up). -type Chunk = { messages: TopicMessage[]; releaseBytes: bigint } +// Read credit bounds encoded bytes. Keep the grant alive until its buffered +// messages are delivered, including after a clean close. +type Chunk = { + messages: Array + releaseBytes: bigint +} +type ReadBatch = { messages: TopicMessage[]; releaseBytes: bigint } type CommitBatch = { messages: TopicMessage[] completion: PromiseWithResolvers promise: Promise - flushed: boolean } // Bind the read offsets to the transaction via UpdateOffsetsInTransaction, so they @@ -70,7 +74,8 @@ let commitTxOffsets = async function commitTxOffsets( tx: TX, driver: Driver, consumer: string, - updates: TxReadOffsetUpdate[] + updates: TxReadOffsetUpdate[], + signal?: AbortSignal ): Promise { if (updates.length === 0) { return @@ -116,7 +121,7 @@ let commitTxOffsets = async function commitTxOffsets( dbg.log('committing read offsets in tx %s (%d partitions)', tx.transactionId, updates.length) let client = driver.createClient(TopicServiceDefinition, tx.nodeId) - let response = await client.updateOffsetsInTransaction(request) + let response = await client.updateOffsetsInTransaction(request, signal ? { signal } : undefined) if (response.operation?.status !== StatusIds_StatusCode.SUCCESS) { // YDBError carries the status code and issues, so retry classifiers and user // code can inspect it like every other server-status failure in the package. @@ -131,7 +136,7 @@ let commitTxOffsets = async function commitTxOffsets( // commit() acknowledges offsets and resolves once the server's committed high-water // mark reaches them — surviving transparent reconnects (never rejected by one). export class TopicReader implements AsyncDisposable, Disposable { - #options: TopicReaderOptions + #callbacks: Pick #codecs: CodecMap #runtime: ReaderRuntime @@ -163,19 +168,27 @@ export class TopicReader implements AsyncDisposable, Disposable { // (or a machine fault), then consulted synchronously by read()/commit()/close() to // surface it to the caller — the FSM cannot reject an already-running read() promise. #lastError: unknown = undefined - #closed = false #closing = false #reading = false // read() is single-consumer - #transactional: boolean #scope: ReaderScope #closedDeferred = Promise.withResolvers() constructor(driver: Driver, options: TopicReaderOptions, runtimeOptions?: { tx?: TX }) { - this.#options = options - this.#transactional = runtimeOptions?.tx !== undefined - if (this.#transactional) { + if (options.maxBufferBytes !== undefined && options.maxBufferBytes <= 0n) { + throw new RangeError('maxBufferBytes must be positive') + } + options = { ...options } + this.#callbacks = { + ...(options.onPartitionSessionStop && { + onPartitionSessionStop: options.onPartitionSessionStop, + }), + ...(options.onCommittedOffset && { onCommittedOffset: options.onCommittedOffset }), + } + + if (runtimeOptions?.tx !== undefined) { this.#txReadOffsets = new Map() } + this.#codecs = options.codecMap ?? defaultCodecMap this.#scope = { driver: driver.identity, @@ -198,14 +211,15 @@ export class TopicReader implements AsyncDisposable, Disposable { // #-private state directly instead of going through an exported accessor. if (runtimeOptions?.tx) { let tx = runtimeOptions.tx - tx.onCommit(async () => { + tx.onCommit(async (signal) => { // Bind the read offsets to the tx; on failure the commit (and thus the // offsets) roll back, and the reader is torn down by onClose below. await commitTxOffsets( tx, driver, - options.consumer, - txOffsetUpdates(this.#txReadOffsets) + this.#scope.consumer, + txOffsetUpdates(this.#txReadOffsets), + signal ) // Release the partition once offsets are committed. A tx reader left open // keeps the consumer's partition assigned server-side, so a later reader on @@ -214,9 +228,11 @@ export class TopicReader implements AsyncDisposable, Disposable { await this.close() }) tx.onRollback(() => { + this.#txReadOffsets?.clear() this.destroy(new Error('Transaction rolled back')) }) tx.onClose((committed) => { + this.#txReadOffsets?.clear() if (!committed) { this.destroy(new Error('Transaction closed without commit')) } @@ -246,6 +262,7 @@ export class TopicReader implements AsyncDisposable, Disposable { `batchWindowMs must be a non-negative finite number, got ${batchWindowMs}` ) } + return this.#readLoop(limit, batchWindowMs, options?.signal) } @@ -253,22 +270,24 @@ export class TopicReader implements AsyncDisposable, Disposable { // The TopicTxReader type hides commit(), but the method still exists on the // runtime object — enforce the boundary for plain-JS callers too: a manual // commit would land outside the transaction and survive its rollback. - if (this.#transactional) { + if (this.#txReadOffsets !== undefined) { throw new Error( 'Tx reader commits offsets via the transaction — commit() is not available' ) } + let batch = this.#commitBatch ?? this.#createCommitBatch() let messages = Array.isArray(input) ? input : [input] for (let message of messages) { batch.messages.push(message) } + return batch.promise } async close(): Promise { - if (this.#closed) { - if (this.#lastError) { + if (this.#chunks.isClosed) { + if (this.#lastError !== undefined) { throw this.#lastError } return @@ -278,21 +297,26 @@ export class TopicReader implements AsyncDisposable, Disposable { if (this.#commitBatch) { this.#flushCommitBatch(this.#commitBatch) } + this.#closing = true this.#runtime.machine.dispatch({ type: 'reader.close' }) await this.#closedDeferred.promise - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } } destroy(reason?: unknown): void { - if (this.#closed) { + if (this.#chunks.isClosed) { + this.#discardBuffers() return } + this.#closing = true let error = reason ?? new Error('Reader destroyed') - this.#lastError = error + this.#lastError ??= error + this.#txReadOffsets?.clear() + this.#discardBuffers() this.#runtime.machine.dispatch({ type: 'reader.destroy', reason: error }) } @@ -312,7 +336,7 @@ export class TopicReader implements AsyncDisposable, Disposable { // Debuggers and util.inspect show the constructor name, which cannot tell a tx // reader apart — the tag makes it render as TopicReader [TopicTxReader] { ... }. get [Symbol.toStringTag](): string { - return this.#transactional ? 'TopicTxReader' : 'TopicReader' + return this.#txReadOffsets !== undefined ? 'TopicTxReader' : 'TopicReader' } async *#readLoop( @@ -325,13 +349,15 @@ export class TopicReader implements AsyncDisposable, Disposable { if (this.#reading) { throw new Error('read() is already in progress — the reader is single-consumer') } + this.#reading = true try { for (;;) { - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } + signal?.throwIfAborted() // Accumulate a batch of up to `limit` messages, waiting at most // `batchWindowMs` (a batch is yielded — possibly empty — once the window @@ -343,7 +369,7 @@ export class TopicReader implements AsyncDisposable, Disposable { let batchLength = chunks.reduce((count, chunk) => count + chunk.messages.length, 0) let closed = false - try { + { // The batch window is a cancellation source: link it with the user signal // so take() aborts when either fires. linkSignals (not the banned // AbortSignal.any) releases its listeners at batch end via `using`. No @@ -372,20 +398,33 @@ export class TopicReader implements AsyncDisposable, Disposable { if (signal?.aborted) { throw signal.reason } + if (waitSignal?.aborted) { break } + throw error } + + if (this.#lastError !== undefined) { + if (!result.done) { + result.value.messages.length = 0 + } + + throw this.#lastError + } + if (result.done) { closed = true break } + let chunk = this.#filterChunk(result.value) if (chunk) { chunks.push(chunk) batchLength += chunk.messages.length } + if ( (limit !== undefined && batchLength >= limit) || batchWindowMs === undefined @@ -395,29 +434,30 @@ export class TopicReader implements AsyncDisposable, Disposable { } if (batchLength > 0) { - while (batchLength > 0) { + while (chunks.length > 0) { + signal?.throwIfAborted() let batch = this.#takeBatch(chunks, limit) - batchLength -= batch.messages.length + if (this.#lastError !== undefined) { + throw this.#lastError + } growTxOffsets(this.#txReadOffsets, batch.messages) if (batch.releaseBytes > 0n) { this.#releaseReadBytes(batch.releaseBytes) } - yield batch.messages + + if (batch.messages.length > 0) { + yield batch.messages + } } } else if (batchWindowMs !== undefined && !closed) { // Idle-window tick: yield an empty batch so the consumer can act. yield [] } - } finally { - this.#carry.push(...chunks) } if (closed) { - // A terminal error surfaces to the consumer — it must not look like a - // clean end-of-stream. Any buffered batch above was delivered first, then - // we throw. The reader is already torn down (markClosed + FSM finalize): - // it is not reusable, and every further read()/commit() throws this too. - if (this.#lastError) { + // A terminal error must not look like a clean end-of-stream. + if (this.#lastError !== undefined) { throw this.#lastError } return @@ -435,11 +475,14 @@ export class TopicReader implements AsyncDisposable, Disposable { } publishBufferChanged(this.#scope, this.#bufferedBytes) this.#runtime.machine.dispatch({ type: 'reader.read_release', bytes }) + if (this.#chunks.isClosed && this.#bufferedBytes === 0n) { + this.#codecs = defaultCodecMap + } } // Remove up to `limit` messages while retaining the response boundary. Credit for // a response is returned only by the batch containing its final message. - #takeBatch(chunks: Chunk[], limit: number | undefined): Chunk { + #takeBatch(chunks: Chunk[], limit: number | undefined): ReadBatch { let messages: TopicMessage[] = [] let releaseBytes = 0n let remaining = limit ?? Infinity @@ -447,47 +490,52 @@ export class TopicReader implements AsyncDisposable, Disposable { let chunk = chunks[0]! let count = Math.min(chunk.messages.length, remaining) for (let message of chunk.messages.splice(0, count)) { - messages.push(message) + if (!message.session.isStopped) { + try { + messages.push(toTopicMessage(this.#codecs, message.session, message)) + } catch (error) { + this.#lastError ??= error + this.#txReadOffsets?.clear() + this.destroy(error) + throw error + } + remaining-- + } } - remaining -= count + if (chunk.messages.length === 0) { releaseBytes += chunk.releaseBytes chunks.shift() } } + return { messages, releaseBytes } } // A force-stopped partition is re-read by its new owner. If filtering consumes the // whole response, there is no user-visible message to gate its credit on. #filterChunk(chunk: Chunk): Chunk | undefined { - let messages = chunk.messages.filter((message) => { - let session = message.partitionSession.deref() - return session !== undefined && !session.isStopped - }) - if (messages.length === 0) { + chunk.messages = chunk.messages.filter((message) => !message.session.isStopped) + if (chunk.messages.length === 0) { this.#releaseReadBytes(chunk.releaseBytes) return undefined } - return { messages, releaseBytes: chunk.releaseBytes } + + return chunk } - // Drain the carry-over, re-applying the stopped-session filter — a partition may - // have been lost between the aborted call and this one. + // A partition may have been lost between the aborted call and this one. #takeCarry(): Chunk[] { - if (this.#carry.length === 0) { - return [] - } - let carried = this.#carry - this.#carry = [] - let chunks: Chunk[] = [] - for (let chunk of carried) { - let filtered = this.#filterChunk(chunk) + for (let i = 0; i < this.#carry.length; ) { + let filtered = this.#filterChunk(this.#carry[i]!) if (filtered) { - chunks.push(filtered) + this.#carry[i++] = filtered + } else { + this.#carry.splice(i, 1) } } - return chunks + + return this.#carry } #createCommitBatch(): CommitBatch { @@ -496,21 +544,19 @@ export class TopicReader implements AsyncDisposable, Disposable { messages: [], completion, promise: traceCommit(this.#scope, () => completion.promise), - flushed: false, } + this.#commitBatch = batch queueMicrotask(() => this.#flushCommitBatch(batch)) return batch } #flushCommitBatch(batch: CommitBatch): void { - if (batch.flushed) { + if (this.#commitBatch !== batch) { return } - batch.flushed = true - if (this.#commitBatch === batch) { - this.#commitBatch = undefined - } + + this.#commitBatch = undefined void this.#commitOffsets(batch.messages).then( batch.completion.resolve, batch.completion.reject @@ -518,10 +564,11 @@ export class TopicReader implements AsyncDisposable, Disposable { } async #commitOffsets(messages: TopicMessage[]): Promise { - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } - if (this.#closed || this.#closing) { + + if (this.#chunks.isClosed || this.#closing) { throw new Error('Reader is closed — cannot commit') } @@ -581,9 +628,11 @@ export class TopicReader implements AsyncDisposable, Disposable { errors.push(result.reason) } } + if (errors.length === 1) { throw errors[0] } + if (errors.length > 1) { throw new AggregateError(errors, 'Cannot commit one or more partitions') } @@ -597,23 +646,20 @@ export class TopicReader implements AsyncDisposable, Disposable { case 'reader.messages': { // tx read-offset tracking happens at read() yield time, not here — a tx // must never bind offsets of messages the consumer never saw. - try { - let messages: TopicMessage[] = [] - for (let group of output.groups) { - for (let message of group.messages) { - messages.push(toTopicMessage(this.#codecs, group.session, message)) - } + if (this.#lastError !== undefined) { + break + } + + let messages: Chunk['messages'] = [] + for (let group of output.groups) { + for (let message of group.messages) { + messages.push({ ...message, session: group.session }) } - this.#bufferedBytes += output.releaseBytes - publishBufferChanged(this.#scope, this.#bufferedBytes) - this.#chunks.push({ messages, releaseBytes: output.releaseBytes }) - } catch (error) { - // An undecodable message (corrupt payload / unsupported codec) - // faults the reader — tear the machine down cleanly rather than - // crash the drain loop into an unhandled rejection. - this.#lastError ??= error - this.#runtime.machine.dispatch({ type: 'reader.destroy', reason: error }) } + + this.#bufferedBytes += output.releaseBytes + publishBufferChanged(this.#scope, this.#bufferedBytes) + this.#chunks.push({ messages, releaseBytes: output.releaseBytes }) break } @@ -634,14 +680,14 @@ export class TopicReader implements AsyncDisposable, Disposable { // end-of-partition are after-the-fact notifications. if ( (output.reason === 'lost' || output.reason === 'ended') && - this.#options.onPartitionSessionStop + this.#callbacks.onPartitionSessionStop ) { // Callback errors are logged via dbg and ignored — a throwing user // callback must never break the machine. Async rejections are caught // the same way. try { Promise.resolve( - this.#options.onPartitionSessionStop( + this.#callbacks.onPartitionSessionStop( output.session, output.session.partitionCommittedOffset ) @@ -659,11 +705,19 @@ export class TopicReader implements AsyncDisposable, Disposable { // watermark carried by a stop request, and commitOffset overrides alike; // also for ended/stopped partitions (a consumer tracking offsets // externally needs the final ack). - if (this.#options.onCommittedOffset) { + if (this.#callbacks.onCommittedOffset) { // Callback errors are logged via dbg and ignored — a throwing user // callback must never break the machine. try { - this.#options.onCommittedOffset(output.session, output.committedOffset) + let result: unknown = this.#callbacks.onCommittedOffset( + output.session, + output.committedOffset + ) + if (isPromise(result)) { + void result.catch((error) => + dbg.log('onCommittedOffset threw: %O', error) + ) + } } catch (error) { dbg.log('onCommittedOffset threw: %O', error) } @@ -698,6 +752,7 @@ export class TopicReader implements AsyncDisposable, Disposable { case 'reader.error': dbg.log('errored: %O', output.error) this.#lastError = output.error + this.#discardBuffers() publishErrored(this.#scope, output.error) break @@ -715,7 +770,7 @@ export class TopicReader implements AsyncDisposable, Disposable { } // Stream ended; if no reader.closed arrived, the machine faulted — surface it. - if (!this.#closed) { + if (!this.#chunks.isClosed) { this.#fail( this.#runtime.machine.signal.reason ?? new Error('Reader stopped unexpectedly') ) @@ -723,11 +778,17 @@ export class TopicReader implements AsyncDisposable, Disposable { } #markClosed(): void { - if (this.#closed) { - return + if (this.#lastError !== undefined) { + this.#txReadOffsets?.clear() + this.#discardBuffers() + } else { + this.#chunks.close() + if (this.#bufferedBytes === 0n) { + this.#codecs = defaultCodecMap + } } - this.#closed = true - this.#chunks.close() + + this.#callbacks = {} // #txReadOffsets is deliberately kept: a tx reader closed before the tx commits // still binds the offsets it delivered — clearing here would silently commit // the transaction with no offsets and redeliver everything after it. @@ -736,17 +797,26 @@ export class TopicReader implements AsyncDisposable, Disposable { for (let waiter of this.#waiters.values()) { waiter.reject(this.#lastError ?? new Error('Reader closed')) } + this.#waiters.clear() this.#closedDeferred.resolve() } #fail(error: unknown): void { - if (this.#closed) { - return - } this.#lastError ??= error this.#markClosed() } + + #discardBuffers(): void { + this.#chunks.destroy() + for (let chunk of this.#carry) chunk.messages.length = 0 + this.#carry.length = 0 + this.#codecs = defaultCodecMap + if (this.#bufferedBytes !== 0n) { + this.#bufferedBytes = 0n + publishBufferChanged(this.#scope, 0n) + } + } } export function createTopicReader(driver: Driver, options: TopicReaderOptions): TopicReader { diff --git a/packages/topic/src/reader/reader.tx-semantics.test.ts b/packages/topic/src/reader/reader.tx-semantics.test.ts index 866ba2d4..aca93cde 100644 --- a/packages/topic/src/reader/reader.tx-semantics.test.ts +++ b/packages/topic/src/reader/reader.tx-semantics.test.ts @@ -195,20 +195,77 @@ test('preserves tracked read offsets when the reader closes before the tx commit ) await collect(reader, 1, tc.signal) - let closeError: unknown - try { - await reader.close() - } catch (error) { - closeError = error - } - let commitError: unknown - try { - await fake.commit() - } catch (error) { - commitError = error - } + await expect(reader.close()).resolves.toBeUndefined() + await expect(fake.commit()).resolves.toBeUndefined() + expect(txOffsetRequests).toHaveLength(1) + expect(txOffsetRequests[0]!.topics[0]!.partitions[0]!.partitionOffsets).toEqual([ + expect.objectContaining({ start: 0n, end: 1n }), + ]) +}) - expect( - closeError !== undefined || commitError !== undefined || txOffsetRequests.length === 1 - ).toBe(true) +test('keeps the transaction consumer registered at reader creation', async (tc) => { + let { driver, waitForNextStream, txOffsetRequests } = makeFakeTopicDriver() + let fake = makeFakeTx() + let options = { topic: '/t', consumer: 'registered' } + using reader = new TopicReader(driver, options, { tx: fake.tx }) + options.consumer = 'replacement' + let stream = await primeStream(waitForNextStream) + expect((await stream.waitForInit()).consumer).toBe('registered') + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('a') }], + }) + ) + await collect(reader, 1, tc.signal) + await fake.commit() + expect(txOffsetRequests).toHaveLength(1) + expect(txOffsetRequests[0]!.consumer).toBe('registered') +}) + +// The stalled unary call makes cancellation observable before a server response; +// a live server cannot guarantee that ordering deterministically. +test('cancels a pending transaction-offset request with the commit hook signal', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let entered = Promise.withResolvers() + let innerCreateClient = driver.createClient.bind(driver) as (...args: unknown[]) => object + ;(driver as unknown as { createClient: (...args: unknown[]) => unknown }).createClient = ( + ...args + ) => ({ + ...innerCreateClient(...args), + updateOffsetsInTransaction(_request: unknown, options?: { signal?: AbortSignal }) { + entered.resolve() + let signal = options?.signal + if (!signal) { + return Promise.reject(new Error('Missing transaction commit signal')) + } + return new Promise((_, reject) => { + if (signal.aborted) { + reject(signal.reason) + } else { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + } + }) + }, + }) + let fake = makeFakeTx() + using reader = new TopicReader(driver, { topic: '/t', consumer: 'c' }, { tx: fake.tx }) + let stream = await primeStream(waitForNextStream) + stream.respond(startPartitionSession({ partitionSessionId: 1n, partitionId: 10n })) + await stream.waitForStartResponse() + stream.respond( + readResponse({ + partitionSessionId: 1n, + messages: [{ offset: 0n, seqNo: 1n, data: bytes('a') }], + }) + ) + await collect(reader, 1, tc.signal) + let ctrl = new AbortController() + let reason = new Error('Transaction cancelled') + let outcome = Promise.resolve(fake.commit(ctrl.signal)).catch((error: unknown) => error) + await entered.promise + ctrl.abort(reason) + expect(await outcome).toBe(reason) }) diff --git a/packages/topic/src/reader/transport-state.test.ts b/packages/topic/src/reader/transport-state.test.ts deleted file mode 100644 index 811c4051..00000000 --- a/packages/topic/src/reader/transport-state.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { expect, test } from 'vitest' - -import type { StreamReadMessage_FromServer } from '@ydbjs/api/topic' -import type { TransitionRuntime } from '@ydbjs/fsm' - -import { - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.ts' - -type Driven = { - state: TransportState - effects: TransportEffect[] - emitted: TransportOutput[] -} - -let drive = function drive(state: TransportState, event: TransportEvent): Driven { - let emitted: TransportOutput[] = [] - let runtime: TransitionRuntime = { - state, - signal: new AbortController().signal, - emit: (output) => emitted.push(output), - dispatch: () => {}, - } - - let result = transportTransition({}, event, runtime) - - return { - state: result?.state ?? state, - effects: result?.effects ?? [], - emitted, - } -} - -let effectTypes = function effectTypes(effects: TransportEffect[]): string[] { - return effects.map((effect) => effect.type) -} - -// Minimal server message — the transition forwards it verbatim, so the shape -// beyond `serverMessage.case` is irrelevant here. -let serverMessage = function serverMessage(id: bigint): StreamReadMessage_FromServer { - return { - serverMessage: { case: 'readResponse', value: { partitionData: [], bytesSize: id } }, - } as unknown as StreamReadMessage_FromServer -} - -// ── idle ──────────────────────────────────────────────────────────────────────── - -test('opens the stream on connect from idle', () => { - let d = drive('idle', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('finalizes on close from idle', () => { - let d = drive('idle', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual(['transport.effect.finalize']) -}) - -test('ignores server messages while idle', () => { - let d = drive('idle', { type: 'transport.message', message: serverMessage(1n) }) - expect(d.state).toBe('idle') - expect(d.effects).toHaveLength(0) - expect(d.emitted).toHaveLength(0) -}) - -// ── connecting ────────────────────────────────────────────────────────────────── - -test('becomes ready and emits the init response', () => { - let d = drive('connecting', { type: 'transport.init', sessionId: 's1' }) - expect(d.state).toBe('ready') - expect(d.emitted).toEqual([{ type: 'transport.stream.init_response', sessionId: 's1' }]) -}) - -test('forwards a server message while connecting without changing state', () => { - let message = serverMessage(2n) - let d = drive('connecting', { type: 'transport.message', message }) - expect(d.state).toBe('connecting') - expect(d.emitted).toEqual([{ type: 'transport.stream.message', message }]) -}) - -test('disconnects on stream end while connecting', () => { - let d = drive('connecting', { type: 'transport.ended' }) - expect(d.state).toBe('disconnected') - expect(effectTypes(d.effects)).toEqual(['transport.effect.close_stream']) - expect(d.emitted).toEqual([{ type: 'transport.stream.disconnected' }]) -}) - -test('reopens the stream on connect while connecting', () => { - let d = drive('connecting', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes while connecting', () => { - let d = drive('connecting', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) -}) - -// ── ready ───────────────────────────────────────────────────────────────────── - -test('forwards a server message while ready', () => { - let message = serverMessage(5n) - let d = drive('ready', { type: 'transport.message', message }) - expect(d.state).toBe('ready') - expect(d.emitted).toEqual([{ type: 'transport.stream.message', message }]) -}) - -test('stays ready on a second init response', () => { - let d = drive('ready', { type: 'transport.init', sessionId: 's2' }) - expect(d.state).toBe('ready') - expect(d.emitted[0]).toMatchObject({ type: 'transport.stream.init_response', sessionId: 's2' }) -}) - -test('disconnects with the error on a stream error while ready', () => { - let error = new Error('boom') - let d = drive('ready', { type: 'transport.error', error }) - expect(d.state).toBe('disconnected') - expect(effectTypes(d.effects)).toEqual(['transport.effect.close_stream']) - expect(d.emitted).toEqual([{ type: 'transport.stream.disconnected', error }]) -}) - -test('reopens the stream on connect while ready', () => { - let d = drive('ready', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes while ready', () => { - let d = drive('ready', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) -}) - -// ── disconnected ──────────────────────────────────────────────────────────────── - -test('reconnects from disconnected on connect', () => { - let d = drive('disconnected', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes from disconnected', () => { - let d = drive('disconnected', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual(['transport.effect.finalize']) -}) - -test('ignores server messages while disconnected', () => { - let d = drive('disconnected', { type: 'transport.message', message: serverMessage(1n) }) - expect(d.state).toBe('disconnected') - expect(d.effects).toHaveLength(0) - expect(d.emitted).toHaveLength(0) -}) - -// ── destroy / closed ──────────────────────────────────────────────────────────── - -test('destroys from any live state', () => { - for (let state of ['idle', 'connecting', 'ready', 'disconnected'] as TransportState[]) { - let d = drive(state, { type: 'transport.destroy', reason: new Error('x') }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) - } -}) - -test('ignores everything once closed', () => { - expect(drive('closed', { type: 'transport.connect' }).state).toBe('closed') - expect(drive('closed', { type: 'transport.destroy' }).effects).toHaveLength(0) - expect( - drive('closed', { type: 'transport.message', message: serverMessage(1n) }).emitted - ).toHaveLength(0) -}) diff --git a/packages/topic/src/reader/transport-state.ts b/packages/topic/src/reader/transport-state.ts deleted file mode 100644 index 9f3b9065..00000000 --- a/packages/topic/src/reader/transport-state.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { fail } from 'node:assert/strict' - -import type { StreamReadMessage_FromServer } from '@ydbjs/api/topic' -import type { TransitionResult, TransitionRuntime } from '@ydbjs/fsm' - -// One physical streamRead lifecycle. All reader business logic (partition -// sessions, offsets, commits, buffer, backoff) lives in the reader FSM; the -// transport only owns the stream and forwards server messages as outputs. -// -// The full transition map (table + diagram) lives in packages/topic/ARCHITECTURE.md — -// update it in the same commit when you change this dispatch. -// -// Unlike the writer transport, the reader has many server-message types -// (read / start-partition / stop-partition / commit / end-partition / …), so -// the transport does not classify them — it recognizes only the init handshake -// and forwards every other server message verbatim for the reader FSM to route. -export type TransportState = 'idle' | 'connecting' | 'ready' | 'disconnected' | 'closed' - -// The transport FSM keeps no ctx state — every fact rides on the event/output. -export type TransportCtx = {} - -export type TransportEvent = - // lifecycle commands the owner dispatches inward - | { type: 'transport.connect' } - | { type: 'transport.close' } - | { type: 'transport.destroy'; reason?: unknown } - // stream facts the ingest task forwards — only the init handshake is - // recognized; every other server message rides verbatim in transport.message - // for the reader FSM to route - | { type: 'transport.init'; sessionId: string } - | { type: 'transport.message'; message: StreamReadMessage_FromServer } - | { type: 'transport.ended' } - | { type: 'transport.error'; error: unknown } - -export type TransportEffect = - | { type: 'transport.effect.open_stream' } - | { type: 'transport.effect.close_stream' } - | { type: 'transport.effect.finalize'; reason: unknown } - -// Emitted to the reader FSM, which ingests these as reader events. -export type TransportOutput = - | { type: 'transport.stream.init_response'; sessionId: string } - | { type: 'transport.stream.message'; message: StreamReadMessage_FromServer } - | { type: 'transport.stream.disconnected'; error?: unknown } - -type TransportRuntime = TransitionRuntime - -let disconnect = function disconnect( - error: unknown, - runtime: TransportRuntime -): TransitionResult { - let output: TransportOutput = { type: 'transport.stream.disconnected' } - if (error !== undefined) { - output = { ...output, error } - } - runtime.emit(output) - return { state: 'disconnected', effects: [{ type: 'transport.effect.close_stream' }] } -} - -export let transportTransition = function transportTransition( - _ctx: TransportCtx, - event: TransportEvent, - runtime: TransportRuntime -): TransitionResult | void { - let state = runtime.state - - if (state !== 'closed' && event.type === 'transport.destroy') { - return { - state: 'closed', - effects: [ - { type: 'transport.effect.close_stream' }, - { - type: 'transport.effect.finalize', - reason: event.reason ?? new Error('Transport destroyed'), - }, - ], - } - } - - switch (state) { - case 'idle': { - if (event.type === 'transport.connect') { - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'connecting': - case 'ready': { - if (event.type === 'transport.connect') { - // Reopen the stream (open_stream disposes the previous one first). - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.init') { - runtime.emit({ type: 'transport.stream.init_response', sessionId: event.sessionId }) - return state === 'ready' ? undefined : { state: 'ready' } - } - if (event.type === 'transport.message') { - runtime.emit({ type: 'transport.stream.message', message: event.message }) - return - } - if (event.type === 'transport.ended') { - return disconnect(undefined, runtime) - } - if (event.type === 'transport.error') { - return disconnect(event.error, runtime) - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { type: 'transport.effect.close_stream' }, - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'disconnected': { - if (event.type === 'transport.connect') { - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'closed': - return - - default: { - let exhaustive: never = state - fail(`Unhandled transport state: ${exhaustive}`) - } - } -} diff --git a/packages/topic/src/reader/transport.ts b/packages/topic/src/reader/transport.ts index e933105e..d37ad80c 100644 --- a/packages/topic/src/reader/transport.ts +++ b/packages/topic/src/reader/transport.ts @@ -3,6 +3,7 @@ import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { type StreamReadMessage_FromClient, StreamReadMessage_FromClientSchema, + type StreamReadMessage_FromServer, StreamReadMessage_InitRequestSchema, type StreamReadMessage_InitRequest_TopicReadSettings, TopicServiceDefinition, @@ -11,17 +12,7 @@ import { import type { Driver } from '@ydbjs/core' import { loggers } from '@ydbjs/debug' import { YDBError } from '@ydbjs/error' -import { type MachineRuntime, createMachineRuntime } from '@ydbjs/fsm' -import { AsyncPriorityQueue } from '@ydbjs/fsm/queue' - -import { - type TransportCtx, - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.js' +import { AsyncPriorityQueue, AsyncQueue } from '@ydbjs/fsm/queue' let dbg = loggers.topic.extend('reader').extend('transport') @@ -42,6 +33,11 @@ export type InitParams = { autoPartitioningSupport?: boolean } +export type TransportOutput = + | { type: 'transport.stream.init_response'; sessionId: string } + | { type: 'transport.stream.message'; message: StreamReadMessage_FromServer } + | { type: 'transport.stream.disconnected'; error?: unknown } + // Owns one streamRead gRPC stream at a time. Reconnecting the underlying stream // is transparent to the reader FSM: each open pushes a fresh init request and the // ingest task forwards server messages (verbatim, except the init handshake) as @@ -50,70 +46,58 @@ export class ReaderTransport { #driver: Driver #params: InitParams - #machine: MachineRuntime + #events = new AsyncQueue() #streamAC: AbortController | null = null #streamInput: AsyncPriorityQueue | null = null - #streamTask: Promise | null = null #tokenPending = false constructor(driver: Driver, params: InitParams) { this.#driver = driver this.#params = params - - this.#machine = createMachineRuntime< - TransportState, - TransportCtx, - {}, - TransportEvent, - TransportEffect, - TransportOutput - >({ - initialState: 'idle', - ctx: {}, - env: {}, - transition: transportTransition, - effects: { - 'transport.effect.open_stream': () => { - this.#openStream() - }, - 'transport.effect.close_stream': async () => { - await this.#closeStream() - }, - 'transport.effect.finalize': async () => { - await this.#closeStream() - }, - }, - }) } // The reader FSM ingests this to receive stream lifecycle events. get events(): AsyncIterable { - return this.#machine + return this.#events } connect(): void { - this.#machine.dispatch({ type: 'transport.connect' }) + if (!this.#events.isClosed) { + this.#openStream() + } } // Enqueue a client message (read request, commit, partition-session response) // on the current stream. No-op if the stream is gone — the reader FSM rebuilds // its outgoing state on the next init anyway. send(message: StreamReadMessage_FromClient, priority: number = PRIORITY_DEFAULT): boolean { - return this.#push(message, priority) + let input = this.#streamInput + if (!input) { + return false + } + + input.push(message, priority) + return true } async sendUpdateToken(): Promise { - // Coalesce: keep at most one un-acknowledged update-token queued (see the - // writer transport for the rationale) — an interval-driven push on a wedged - // stream would otherwise pile up unboundedly. - if (this.#tokenPending) { + let input = this.#streamInput + if (!input || this.#tokenPending) { return } + this.#tokenPending = true + try { let token = await this.#driver.token - let queued = this.#push( + + // A refresh belongs to the stream that requested it, even across token-provider awaits. + if (input !== this.#streamInput) { + return + } + + input.push( create(StreamReadMessage_FromClientSchema, { clientMessage: { case: 'updateTokenRequest', @@ -122,34 +106,26 @@ export class ReaderTransport { }), PRIORITY_TOKEN ) - if (!queued) { - this.#tokenPending = false // nothing landed on the wire; allow a retry - } } catch (error) { - this.#tokenPending = false + if (input === this.#streamInput) { + this.#tokenPending = false + } + throw error } } close(): void { - this.#machine.dispatch({ type: 'transport.close' }) + this.destroy() } destroy(reason?: unknown): void { - this.#machine.dispatch({ type: 'transport.destroy', reason }) - } - - #push(message: StreamReadMessage_FromClient, priority: number): boolean { - let input = this.#streamInput - if (!input || input.isClosed) { - return false - } - input.push(message, priority) - return true + this.#closeStream(reason) + this.#events.destroy() } #openStream(): void { - void this.#closeStream() + this.#closeStream() let ac = new AbortController() let input = new AsyncPriorityQueue() @@ -169,15 +145,20 @@ export class ReaderTransport { PRIORITY_INIT ) - let dispatch = this.#machine.dispatch.bind(this.#machine) + this.#streamAC = ac + this.#streamInput = input + this.#tokenPending = false - let task = (async () => { + void (async () => { try { await this.#driver.ready(ac.signal) + if (ac.signal.aborted) { + return + } let stream = this.#driver .createClient(TopicServiceDefinition) - .streamRead(input, { signal: ac.signal }) + .streamRead(this.#requests(input), { signal: ac.signal }) dbg.log('stream opened (consumer=%s)', this.#params.consumer) @@ -188,11 +169,7 @@ export class ReaderTransport { if (response.status !== StatusIds_StatusCode.SUCCESS) { dbg.log('recv non-success status %d', response.status) - dispatch({ - type: 'transport.error', - error: new YDBError(response.status, response.issues), - }) - return + throw new YDBError(response.status, response.issues) } if (response.serverMessage.case === 'initResponse') { @@ -200,52 +177,57 @@ export class ReaderTransport { 'recv initResponse (sessionId=%s)', response.serverMessage.value.sessionId ) - dispatch({ - type: 'transport.init', + this.#events.push({ + type: 'transport.stream.init_response', sessionId: response.serverMessage.value.sessionId, }) continue } - if (response.serverMessage.case === 'updateTokenResponse') { - this.#tokenPending = false - } - - dispatch({ type: 'transport.message', message: response }) + this.#events.push({ type: 'transport.stream.message', message: response }) } dbg.log('stream ended') - dispatch({ type: 'transport.ended' }) + if (!ac.signal.aborted) { + this.#disconnect() + } } catch (error) { if (ac.signal.aborted) { return } dbg.log('stream error: %O', error) - dispatch({ type: 'transport.error', error }) + this.#disconnect(error) } })() + } - this.#streamAC = ac - this.#streamInput = input - this.#streamTask = task - this.#tokenPending = false // fresh stream — the previous token, if any, is gone + async *#requests(input: AsyncPriorityQueue) { + for await (let request of input) { + // Read sessions may ignore unchanged tokens, so coalesce until send, not ACK. + if ( + input === this.#streamInput && + request.clientMessage.case === 'updateTokenRequest' + ) { + this.#tokenPending = false + } + yield request + } } - async #closeStream(): Promise { + #disconnect(error?: unknown): void { + this.#closeStream() + this.#events.push({ + type: 'transport.stream.disconnected', + ...(error !== undefined && { error }), + }) + } + + #closeStream(reason?: unknown): void { let ac = this.#streamAC let input = this.#streamInput - let task = this.#streamTask - this.#streamAC = null this.#streamInput = null - this.#streamTask = null - - if (!ac) { - return - } - - ac.abort(new Error('Stream disposed')) - input?.close() - await task + ac?.abort(reason ?? new Error('Stream disposed')) + input?.destroy() } } diff --git a/packages/topic/src/reader/types.ts b/packages/topic/src/reader/types.ts index 3480336c..7828ac4a 100644 --- a/packages/topic/src/reader/types.ts +++ b/packages/topic/src/reader/types.ts @@ -30,7 +30,8 @@ export type onPartitionSessionStartCallback = ( // deliverable and committable, and is awaited before the stop response is sent — // the last chance to commit processed offsets before the partition moves to // another reader (bounded by gracefulShutdownTimeoutMs). On a forced stop or -// end-of-partition it is informational: the session is already stopped. +// end-of-partition it is informational. Forced stops invalidate the session; +// an ended partition remains committable until the server stops its session. export type onPartitionSessionStopCallback = ( partitionSession: TopicPartitionSession, committedOffset: bigint diff --git a/packages/topic/src/retry.test.ts b/packages/topic/src/retry.test.ts new file mode 100644 index 00000000..bc055c74 --- /dev/null +++ b/packages/topic/src/retry.test.ts @@ -0,0 +1,21 @@ +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { YDBError } from '@ydbjs/error' +import { ClientError, Status } from 'nice-grpc' +import { expect, test } from 'vitest' + +import { isRetryableTopicError } from './retry.ts' + +test('reconnects live topic streams after a server deadline', () => { + let error = new ClientError('/stream', Status.DEADLINE_EXCEEDED, 'Stream deadline expired') + expect(isRetryableTopicError(error)).toBe(true) +}) + +test('reconnects a reader after an indeterminate idempotent stream failure', () => { + for (let status of [ + StatusIds_StatusCode.SESSION_EXPIRED, + StatusIds_StatusCode.TIMEOUT, + StatusIds_StatusCode.UNDETERMINED, + ]) { + expect(isRetryableTopicError(new YDBError(status, []))).toBe(true) + } +}) diff --git a/packages/topic/src/retry.ts b/packages/topic/src/retry.ts new file mode 100644 index 00000000..223fdcdf --- /dev/null +++ b/packages/topic/src/retry.ts @@ -0,0 +1,31 @@ +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { YDBError } from '@ydbjs/error' +import { isRetryableError, isRetryableStreamError } from '@ydbjs/retry' +import { ClientError, Status } from 'nice-grpc' + +export function isRetryableTopicError(error: unknown, retryOnSchemeError = false): boolean { + if (error === undefined || error === null) { + return true + } + + // Changing streams cannot make an oversized frame fit; quota exhaustion can recover. + if (error instanceof ClientError) { + if (error.code === Status.RESOURCE_EXHAUSTED && /larger than/i.test(error.details)) { + return false + } + if (error.code === Status.DEADLINE_EXCEEDED) { + return true + } + } + + if ( + retryOnSchemeError && + error instanceof YDBError && + error.code === StatusIds_StatusCode.SCHEME_ERROR + ) { + return true + } + + // Sequence numbers deduplicate writes; consumer offsets make read/commit replay safe. + return isRetryableStreamError(error) || isRetryableError(error, true) +} diff --git a/packages/topic/src/transport.test.ts b/packages/topic/src/transport.test.ts new file mode 100644 index 00000000..1e4a0a95 --- /dev/null +++ b/packages/topic/src/transport.test.ts @@ -0,0 +1,240 @@ +import { create } from '@bufbuild/protobuf' +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { + type StreamReadMessage_FromClient, + type StreamReadMessage_FromServer, + StreamWriteMessage_FromServerSchema, +} from '@ydbjs/api/topic' +import { AsyncQueue } from '@ydbjs/fsm/queue' +import { expect, test, vi } from 'vitest' + +import { + makeFakeTopicDriver as makeReaderDriver, + initResponse as readInit, + settle, + updateTokenResponse, +} from './reader/reader.fixtures.ts' +import { ReaderTransport } from './reader/transport.ts' +import { + makeFakeTopicDriver as makeWriterDriver, + initResponse as writeInit, +} from './writer/writer.fixtures.ts' +import { WriterTransport } from './writer/transport.ts' + +// A fake controls token resolution and old-stream EOF independently of reconnect. +// Real YDB cannot deterministically order these client-side continuations. +let transports = { + reader() { + let fake = makeReaderDriver() + let transport = new ReaderTransport(fake.driver, { consumer: 'c', topicsReadSettings: [] }) + return { + driver: fake.driver, + transport, + connect: () => transport.connect(), + async nextStream() { + let stream = await fake.waitForNextStream() + await stream.waitForInit() + stream.respond(readInit()) + await settle() + return { ...stream, acknowledgeToken: () => stream.respond(updateTokenResponse()) } + }, + [Symbol.dispose]() { + transport.destroy() + }, + } + }, + writer() { + let fake = makeWriterDriver() + let transport = new WriterTransport(fake.driver, { path: '/t', producerId: 'p' }) + return { + driver: fake.driver, + transport, + connect: () => transport.connect(true), + async nextStream() { + let stream = await fake.waitForNextStream() + await stream.waitForInit() + stream.respond(writeInit(0n)) + await settle() + return { + ...stream, + acknowledgeToken: () => + stream.respond( + create(StreamWriteMessage_FromServerSchema, { + status: StatusIds_StatusCode.SUCCESS, + serverMessage: { case: 'updateTokenResponse', value: {} }, + }) + ), + } + }, + [Symbol.dispose]() { + transport.destroy() + }, + } + }, +} + +for (let [name, make] of Object.entries(transports)) { + test(`${name} discards a token resolved after its stream was replaced`, async () => { + using fixture = make() + let oldToken = Promise.withResolvers() + vi.spyOn(fixture.driver, 'token', 'get') + .mockReturnValueOnce(oldToken.promise) + .mockResolvedValue('current-token') + fixture.connect() + await fixture.nextStream() + let oldRefresh = fixture.transport.sendUpdateToken() + fixture.connect() + let current = await fixture.nextStream() + await fixture.transport.sendUpdateToken() + oldToken.resolve('old-token') + await oldRefresh + await settle() + let updates = current.sent.filter( + (message) => message.clientMessage.case === 'updateTokenRequest' + ) + expect(updates).toHaveLength(1) + expect(updates[0]!.clientMessage.value).toMatchObject({ token: 'current-token' }) + expect(current.wasAborted()).toBe(false) + }) + + test(`${name} keeps current token acquisition pending when an old refresh fails`, async () => { + using fixture = make() + let oldToken = Promise.withResolvers() + let currentToken = Promise.withResolvers() + let token = vi + .spyOn(fixture.driver, 'token', 'get') + .mockReturnValueOnce(oldToken.promise) + .mockReturnValue(currentToken.promise) + fixture.connect() + await fixture.nextStream() + let oldRefresh = fixture.transport.sendUpdateToken().catch(() => {}) + fixture.connect() + let current = await fixture.nextStream() + let currentRefresh = fixture.transport.sendUpdateToken() + oldToken.reject(new Error('Old credentials request failed')) + await oldRefresh + await fixture.transport.sendUpdateToken() + currentToken.resolve('current-token') + await currentRefresh + await settle() + expect(token).toHaveBeenCalledTimes(2) + expect( + current.sent.filter((message) => message.clientMessage.case === 'updateTokenRequest') + ).toHaveLength(1) + }) + + test(`${name} coalesces concurrent token requests`, async () => { + using fixture = make() + let token = vi.spyOn(fixture.driver, 'token', 'get').mockResolvedValue('token') + fixture.connect() + let stream = await fixture.nextStream() + await Promise.all([ + fixture.transport.sendUpdateToken(), + fixture.transport.sendUpdateToken(), + ]) + expect(token).toHaveBeenCalledTimes(1) + stream.acknowledgeToken() + await settle() + await fixture.transport.sendUpdateToken() + await settle() + expect(token).toHaveBeenCalledTimes(2) + expect( + stream.sent.filter((message) => message.clientMessage.case === 'updateTokenRequest') + ).toHaveLength(2) + }) + + test(`${name} keeps a replacement stream alive after the old stream ends`, async () => { + using fixture = make() + fixture.connect() + let old = await fixture.nextStream() + fixture.connect() + let current = await fixture.nextStream() + await settle() + expect(old.wasAborted()).toBe(true) + expect(current.wasAborted()).toBe(false) + }) + + test(`${name} terminates its event iterator when closed`, async () => { + using fixture = make() + fixture.connect() + let stream = await fixture.nextStream() + let events = fixture.transport.events[Symbol.asyncIterator]() + expect((await events.next()).value?.type).toBe('transport.stream.init_response') + let createClient = vi.spyOn(fixture.driver, 'createClient') + fixture.transport.close() + expect(await events.next()).toMatchObject({ done: true }) + expect(stream.wasAborted()).toBe(true) + fixture.connect() + await expect(fixture.transport.sendUpdateToken()).resolves.toBeUndefined() + await settle() + expect(createClient).not.toHaveBeenCalled() + }) + + test(`${name} does not open a stream after close interrupts driver readiness`, async () => { + using fixture = make() + let ready = Promise.withResolvers() + vi.spyOn(fixture.driver, 'ready').mockReturnValue(ready.promise) + let createClient = vi.spyOn(fixture.driver, 'createClient') + fixture.connect() + await settle() + fixture.transport.close() + await settle() + ready.resolve() + await settle() + expect(createClient).not.toHaveBeenCalled() + }) +} + +test('reader sends a new token when the server does not acknowledge an unchanged token', async () => { + using fixture = transports.reader() + vi.spyOn(fixture.driver, 'token', 'get') + .mockResolvedValueOnce('unchanged-token') + .mockResolvedValue('renewed-token') + fixture.connect() + let stream = await fixture.nextStream() + + await fixture.transport.sendUpdateToken() + await settle() + await fixture.transport.sendUpdateToken() + await settle() + + expect( + stream.sent + .filter((message) => message.clientMessage.case === 'updateTokenRequest') + .map((message) => message.clientMessage.value) + ).toMatchObject([{ token: 'unchanged-token' }, { token: 'renewed-token' }]) +}) + +test('reader bounds queued token refreshes while outgoing requests are blocked', async () => { + let fake = makeReaderDriver() + let responses = new AsyncQueue() + let requests: AsyncIterable | undefined + vi.spyOn(fake.driver, 'createClient').mockReturnValue({ + streamRead(input: AsyncIterable) { + requests = input + return responses + }, + } as never) + let token = vi.spyOn(fake.driver, 'token', 'get').mockResolvedValue('token') + let transport = new ReaderTransport(fake.driver, { consumer: 'c', topicsReadSettings: [] }) + using _ = { + [Symbol.dispose]() { + transport.destroy() + responses.destroy() + }, + } + transport.connect() + await settle() + + await transport.sendUpdateToken() + await transport.sendUpdateToken() + await transport.sendUpdateToken() + expect(token).toHaveBeenCalledTimes(1) + + let iterator = requests![Symbol.asyncIterator]() + expect((await iterator.next()).value.clientMessage.case).toBe('initRequest') + expect((await iterator.next()).value.clientMessage.case).toBe('updateTokenRequest') + await transport.sendUpdateToken() + expect(token).toHaveBeenCalledTimes(2) + await iterator.return?.() +}) diff --git a/packages/topic/src/writer/diagnostics.ts b/packages/topic/src/writer/diagnostics.ts index 99cc1403..6aa30c9b 100644 --- a/packages/topic/src/writer/diagnostics.ts +++ b/packages/topic/src/writer/diagnostics.ts @@ -18,8 +18,7 @@ export type WriterScope = { producer: string } -// One-shot config snapshot, published on `opened` so metrics/traces subscribers -// that join later still learn the writer's effective configuration. +// Configuration snapshot carried by the creation-time `opened` event. export type WriterConfig = { codec: number maxInflightCount: number diff --git a/packages/topic/src/writer/index.ts b/packages/topic/src/writer/index.ts index 88730155..764947aa 100644 --- a/packages/topic/src/writer/index.ts +++ b/packages/topic/src/writer/index.ts @@ -1,2 +1,9 @@ export { TopicWriter, createTopicWriter, createTopicTxWriter } from './writer.js' -export type { AckStatus, OnAckCallback, TopicWriterOptions, WriteAck, WriteExtra } from './types.js' +export type { + AckStatus, + OnAckCallback, + TopicTxWriter, + TopicWriterOptions, + WriteAck, + WriteExtra, +} from './types.js' diff --git a/packages/topic/src/writer/producer-id.ts b/packages/topic/src/writer/producer-id.ts deleted file mode 100644 index 9fab920e..00000000 --- a/packages/topic/src/writer/producer-id.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Generates a unique producer id when the caller does not supply one. The -// producer id + seqNo form the server-side dedup key, so it must be stable for -// the lifetime of one writer but distinct across writers. -export function generateProducerId(): string { - let processId = process.pid - let currentTime = new Date().getTime() - let randomSuffix = Math.floor(Math.random() * 1_000_000) - return `producer-${processId}-${currentTime}-${randomSuffix}` -} diff --git a/packages/topic/src/writer/transport-state.test.ts b/packages/topic/src/writer/transport-state.test.ts deleted file mode 100644 index 71cf79db..00000000 --- a/packages/topic/src/writer/transport-state.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { expect, test } from 'vitest' - -import type { TransitionRuntime } from '@ydbjs/fsm' - -import { - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.ts' -import type { WriteAck } from './types.ts' - -type Driven = { - state: TransportState - effects: TransportEffect[] - emitted: TransportOutput[] -} - -let drive = function drive(state: TransportState, event: TransportEvent): Driven { - let emitted: TransportOutput[] = [] - let runtime: TransitionRuntime = { - state, - signal: new AbortController().signal, - emit: (output) => emitted.push(output), - dispatch: () => {}, - } - - let result = transportTransition({}, event, runtime) - - return { - state: result?.state ?? state, - effects: result?.effects ?? [], - emitted, - } -} - -let effectTypes = function effectTypes(effects: TransportEffect[]): string[] { - return effects.map((effect) => effect.type) -} - -let ack = function ack(seqNo: bigint): WriteAck { - return { seqNo, status: 'written' } -} - -// ── idle ──────────────────────────────────────────────────────────────────────── - -test('opens the stream on connect from idle', () => { - let d = drive('idle', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('finalizes on close from idle', () => { - let d = drive('idle', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual(['transport.effect.finalize']) -}) - -test('ignores unrelated events while idle', () => { - let d = drive('idle', { type: 'transport.write', acks: [ack(1n)] }) - expect(d.state).toBe('idle') - expect(d.effects).toHaveLength(0) - expect(d.emitted).toHaveLength(0) -}) - -// ── connecting ────────────────────────────────────────────────────────────────── - -test('becomes ready and emits the init response', () => { - let d = drive('connecting', { type: 'transport.init', sessionId: 's1', lastSeqNo: 7n }) - expect(d.state).toBe('ready') - expect(d.emitted).toEqual([ - { type: 'transport.stream.init_response', sessionId: 's1', lastSeqNo: 7n }, - ]) -}) - -test('carries the partitionId in the init response', () => { - let d = drive('connecting', { - type: 'transport.init', - sessionId: 's1', - lastSeqNo: 0n, - partitionId: 3n, - }) - expect(d.emitted[0]).toMatchObject({ partitionId: 3n }) -}) - -test('emits a write response while connecting without changing state', () => { - let d = drive('connecting', { type: 'transport.write', acks: [ack(1n), ack(2n)] }) - expect(d.state).toBe('connecting') - expect(d.emitted).toEqual([ - { type: 'transport.stream.write_response', acks: [ack(1n), ack(2n)] }, - ]) -}) - -test('emits a token response while connecting', () => { - let d = drive('connecting', { type: 'transport.token' }) - expect(d.emitted).toEqual([{ type: 'transport.stream.token_response' }]) -}) - -test('disconnects on stream end while connecting', () => { - let d = drive('connecting', { type: 'transport.ended' }) - expect(d.state).toBe('disconnected') - expect(effectTypes(d.effects)).toEqual(['transport.effect.close_stream']) - expect(d.emitted).toEqual([{ type: 'transport.stream.disconnected' }]) -}) - -test('reopens the stream on connect while connecting', () => { - let d = drive('connecting', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes while connecting', () => { - let d = drive('connecting', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) -}) - -// ── ready ───────────────────────────────────────────────────────────────────── - -test('emits a write response while ready', () => { - let d = drive('ready', { type: 'transport.write', acks: [ack(5n)] }) - expect(d.state).toBe('ready') - expect(d.emitted).toEqual([{ type: 'transport.stream.write_response', acks: [ack(5n)] }]) -}) - -test('stays ready on a second init response', () => { - let d = drive('ready', { type: 'transport.init', sessionId: 's2', lastSeqNo: 1n }) - expect(d.state).toBe('ready') - expect(d.emitted[0]).toMatchObject({ type: 'transport.stream.init_response', sessionId: 's2' }) -}) - -test('disconnects with the error on a stream error while ready', () => { - let error = new Error('boom') - let d = drive('ready', { type: 'transport.error', error }) - expect(d.state).toBe('disconnected') - expect(effectTypes(d.effects)).toEqual(['transport.effect.close_stream']) - expect(d.emitted).toEqual([{ type: 'transport.stream.disconnected', error }]) -}) - -test('reopens the stream on connect while ready', () => { - let d = drive('ready', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes while ready', () => { - let d = drive('ready', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) -}) - -// ── disconnected ──────────────────────────────────────────────────────────────── - -test('reconnects from disconnected on connect', () => { - let d = drive('disconnected', { type: 'transport.connect' }) - expect(d.state).toBe('connecting') - expect(effectTypes(d.effects)).toEqual(['transport.effect.open_stream']) -}) - -test('closes from disconnected', () => { - let d = drive('disconnected', { type: 'transport.close' }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual(['transport.effect.finalize']) -}) - -test('ignores stream messages while disconnected', () => { - let d = drive('disconnected', { type: 'transport.write', acks: [ack(1n)] }) - expect(d.state).toBe('disconnected') - expect(d.effects).toHaveLength(0) - expect(d.emitted).toHaveLength(0) -}) - -// ── destroy / closed ──────────────────────────────────────────────────────────── - -test('destroys from any live state', () => { - for (let state of ['idle', 'connecting', 'ready', 'disconnected'] as TransportState[]) { - let d = drive(state, { type: 'transport.destroy', reason: new Error('x') }) - expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toEqual([ - 'transport.effect.close_stream', - 'transport.effect.finalize', - ]) - } -}) - -test('ignores everything once closed', () => { - expect(drive('closed', { type: 'transport.connect' }).state).toBe('closed') - expect(drive('closed', { type: 'transport.destroy' }).effects).toHaveLength(0) - expect(drive('closed', { type: 'transport.write', acks: [ack(1n)] }).emitted).toHaveLength(0) -}) diff --git a/packages/topic/src/writer/transport-state.ts b/packages/topic/src/writer/transport-state.ts deleted file mode 100644 index ac483b94..00000000 --- a/packages/topic/src/writer/transport-state.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { fail } from 'node:assert/strict' - -import type { TransitionResult, TransitionRuntime } from '@ydbjs/fsm' - -import type { WriteAck } from './types.js' - -// One physical streamWrite lifecycle. All business logic (buffer, seqNo, backoff) -// lives in the writer FSM; the transport only owns the socket and forwards -// classified server messages to the writer as outputs. -// -// The full transition map (table + diagram) lives in packages/topic/ARCHITECTURE.md — -// update it in the same commit when you change this dispatch. -export type TransportState = 'idle' | 'connecting' | 'ready' | 'disconnected' | 'closed' - -// The transport FSM keeps no ctx state — every fact rides on the event/output. -export type TransportCtx = {} - -// Events are pre-classified by the ingest task, so the transition never touches -// raw protobuf — it just maps stream facts to state and outputs. -export type TransportEvent = - // lifecycle commands the owner dispatches inward - | { type: 'transport.connect' } - | { type: 'transport.close' } - | { type: 'transport.destroy'; reason?: unknown } - // classified stream facts the ingest task forwards - | { - type: 'transport.init' - sessionId: string - lastSeqNo: bigint - partitionId?: bigint - supportedCodecs?: number[] - } - | { type: 'transport.write'; acks: WriteAck[] } - | { type: 'transport.token' } - | { type: 'transport.ended' } - | { type: 'transport.error'; error: unknown } - -export type TransportEffect = - | { type: 'transport.effect.open_stream' } - | { type: 'transport.effect.close_stream' } - | { type: 'transport.effect.finalize'; reason: unknown } - -// Emitted to the writer FSM, which ingests these as writer events. -export type TransportOutput = - | { - type: 'transport.stream.init_response' - sessionId: string - lastSeqNo: bigint - partitionId?: bigint - supportedCodecs?: number[] - } - | { type: 'transport.stream.write_response'; acks: WriteAck[] } - | { type: 'transport.stream.token_response' } - | { type: 'transport.stream.disconnected'; error?: unknown } - -type TransportRuntime = TransitionRuntime - -let toInitResponse = function toInitResponse( - event: Extract -): TransportOutput { - let output: TransportOutput = { - type: 'transport.stream.init_response', - sessionId: event.sessionId, - lastSeqNo: event.lastSeqNo, - } - if (event.partitionId !== undefined) { - output = { ...output, partitionId: event.partitionId } - } - if (event.supportedCodecs !== undefined) { - output = { ...output, supportedCodecs: event.supportedCodecs } - } - return output -} - -let disconnect = function disconnect( - error: unknown, - runtime: TransportRuntime -): TransitionResult { - let output: TransportOutput = { type: 'transport.stream.disconnected' } - if (error !== undefined) { - output = { ...output, error } - } - runtime.emit(output) - return { state: 'disconnected', effects: [{ type: 'transport.effect.close_stream' }] } -} - -export let transportTransition = function transportTransition( - _ctx: TransportCtx, - event: TransportEvent, - runtime: TransportRuntime -): TransitionResult | void { - let state = runtime.state - - if (state !== 'closed' && event.type === 'transport.destroy') { - return { - state: 'closed', - effects: [ - { type: 'transport.effect.close_stream' }, - { - type: 'transport.effect.finalize', - reason: event.reason ?? new Error('Transport destroyed'), - }, - ], - } - } - - switch (state) { - case 'idle': { - if (event.type === 'transport.connect') { - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'connecting': - case 'ready': { - if (event.type === 'transport.connect') { - // Reopen the stream (open_stream disposes the previous one first). - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.init') { - runtime.emit(toInitResponse(event)) - return state === 'ready' ? undefined : { state: 'ready' } - } - if (event.type === 'transport.write') { - runtime.emit({ type: 'transport.stream.write_response', acks: event.acks }) - return - } - if (event.type === 'transport.token') { - runtime.emit({ type: 'transport.stream.token_response' }) - return - } - if (event.type === 'transport.ended') { - return disconnect(undefined, runtime) - } - if (event.type === 'transport.error') { - return disconnect(event.error, runtime) - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { type: 'transport.effect.close_stream' }, - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'disconnected': { - if (event.type === 'transport.connect') { - return { state: 'connecting', effects: [{ type: 'transport.effect.open_stream' }] } - } - if (event.type === 'transport.close') { - return { - state: 'closed', - effects: [ - { - type: 'transport.effect.finalize', - reason: new Error('Transport closed'), - }, - ], - } - } - return - } - - case 'closed': - return - - default: { - let exhaustive: never = state - fail(`Unhandled transport state: ${exhaustive}`) - } - } -} diff --git a/packages/topic/src/writer/transport.ts b/packages/topic/src/writer/transport.ts index 39f8fb93..1d125ae0 100644 --- a/packages/topic/src/writer/transport.ts +++ b/packages/topic/src/writer/transport.ts @@ -12,17 +12,8 @@ import { import type { Driver } from '@ydbjs/core' import { loggers } from '@ydbjs/debug' import { YDBError } from '@ydbjs/error' -import { type MachineRuntime, createMachineRuntime } from '@ydbjs/fsm' -import { AsyncPriorityQueue } from '@ydbjs/fsm/queue' +import { AsyncPriorityQueue, AsyncQueue } from '@ydbjs/fsm/queue' -import { - type TransportCtx, - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.js' import type { AckStatus, WriteAck } from './types.js' let dbg = loggers.topic.extend('writer').extend('transport') @@ -57,6 +48,18 @@ let flattenAcks = function flattenAcks(response: StreamWriteMessage_WriteRespons }) } +export type TransportOutput = + | { + type: 'transport.stream.init_response' + sessionId: string + lastSeqNo: bigint + partitionId?: bigint + supportedCodecs?: number[] + } + | { type: 'transport.stream.write_response'; acks: WriteAck[] } + | { type: 'transport.stream.token_response' } + | { type: 'transport.stream.disconnected'; error?: unknown } + // Owns one streamWrite gRPC stream at a time. Reconnecting the underlying stream // is transparent to the writer FSM: each open pushes a fresh init request and the // ingest task forwards classified server messages as transport outputs. @@ -64,56 +67,35 @@ export class WriterTransport { #driver: Driver #params: InitParams - #machine: MachineRuntime + #events = new AsyncQueue() #streamAC: AbortController | null = null #streamInput: AsyncPriorityQueue | null = null - #streamTask: Promise | null = null - #getLastSeqNo = true #tokenPending = false constructor(driver: Driver, params: InitParams) { this.#driver = driver this.#params = params - - this.#machine = createMachineRuntime< - TransportState, - TransportCtx, - {}, - TransportEvent, - TransportEffect, - TransportOutput - >({ - initialState: 'idle', - ctx: {}, - env: {}, - transition: transportTransition, - effects: { - 'transport.effect.open_stream': () => { - this.#openStream() - }, - 'transport.effect.close_stream': async () => { - await this.#closeStream() - }, - 'transport.effect.finalize': async () => { - await this.#closeStream() - }, - }, - }) } // The writer FSM ingests this to receive stream lifecycle events. get events(): AsyncIterable { - return this.#machine + return this.#events } connect(getLastSeqNo: boolean): void { - this.#getLastSeqNo = getLastSeqNo - this.#machine.dispatch({ type: 'transport.connect' }) + if (!this.#events.isClosed) { + this.#openStream(getLastSeqNo) + } } sendBatch(request: StreamWriteMessage_WriteRequest): void { - this.#push( + let input = this.#streamInput + if (!input) { + return + } + + input.push( create(StreamWriteMessage_FromClientSchema, { clientMessage: { case: 'writeRequest', value: request }, }), @@ -122,18 +104,22 @@ export class WriterTransport { } async sendUpdateToken(): Promise { - // Coalesce: keep at most one un-acknowledged update-token queued. Data writes - // self-limit at maxInflightCount, but this interval-driven push has no such - // gate — on a wedged-but-open stream (server stopped reading) it would pile - // token frames into #streamInput without bound. Set the flag before awaiting - // the token so a second interval can't slip a duplicate through the gap. - if (this.#tokenPending) { + let input = this.#streamInput + if (!input || this.#tokenPending) { return } + this.#tokenPending = true + try { let token = await this.#driver.token - let queued = this.#push( + + // A refresh belongs to the stream that requested it, even across token-provider awaits. + if (input !== this.#streamInput) { + return + } + + input.push( create(StreamWriteMessage_FromClientSchema, { clientMessage: { case: 'updateTokenRequest', @@ -142,34 +128,26 @@ export class WriterTransport { }), PRIORITY_TOKEN ) - if (!queued) { - this.#tokenPending = false // nothing landed on the wire; allow a retry - } } catch (error) { - this.#tokenPending = false + if (input === this.#streamInput) { + this.#tokenPending = false + } + throw error } } close(): void { - this.#machine.dispatch({ type: 'transport.close' }) + this.destroy() } destroy(reason?: unknown): void { - this.#machine.dispatch({ type: 'transport.destroy', reason }) + this.#closeStream(reason) + this.#events.destroy() } - #push(message: StreamWriteMessage_FromClient, priority: number): boolean { - let input = this.#streamInput - if (!input || input.isClosed) { - return false - } - input.push(message, priority) - return true - } - - #openStream(): void { - void this.#closeStream() + #openStream(getLastSeqNo: boolean): void { + this.#closeStream() let ac = new AbortController() let input = new AsyncPriorityQueue() @@ -181,7 +159,7 @@ export class WriterTransport { value: create(StreamWriteMessage_InitRequestSchema, { path: this.#params.path, producerId: this.#params.producerId, - getLastSeqNo: this.#getLastSeqNo, + getLastSeqNo, ...(this.#params.messageGroupId !== undefined && { partitioning: { case: 'messageGroupId', @@ -197,17 +175,22 @@ export class WriterTransport { PRIORITY_INIT ) - let dispatch = this.#machine.dispatch.bind(this.#machine) + this.#streamAC = ac + this.#streamInput = input + this.#tokenPending = false - let task = (async () => { + void (async () => { try { await this.#driver.ready(ac.signal) + if (ac.signal.aborted) { + return + } let stream = this.#driver .createClient(TopicServiceDefinition) .streamWrite(input, { signal: ac.signal }) - dbg.log('stream opened (getLastSeqNo=%s)', this.#getLastSeqNo) + dbg.log('stream opened (getLastSeqNo=%s)', getLastSeqNo) for await (let response of stream) { if (ac.signal.aborted) { @@ -216,11 +199,7 @@ export class WriterTransport { if (response.status !== StatusIds_StatusCode.SUCCESS) { dbg.log('recv non-success status %d', response.status) - dispatch({ - type: 'transport.error', - error: new YDBError(response.status, response.issues), - }) - return + throw new YDBError(response.status, response.issues) } switch (response.serverMessage.case) { @@ -231,8 +210,8 @@ export class WriterTransport { value.lastSeqNo, value.sessionId ) - dispatch({ - type: 'transport.init', + this.#events.push({ + type: 'transport.stream.init_response', sessionId: value.sessionId, lastSeqNo: value.lastSeqNo, ...(value.partitionId !== undefined && { @@ -254,13 +233,14 @@ export class WriterTransport { acks.map((a) => a.seqNo) ) } - dispatch({ type: 'transport.write', acks }) + + this.#events.push({ type: 'transport.stream.write_response', acks }) break } case 'updateTokenResponse': dbg.log('recv updateTokenResponse') this.#tokenPending = false - dispatch({ type: 'transport.token' }) + this.#events.push({ type: 'transport.stream.token_response' }) break default: dbg.log('recv unknown server message %s', response.serverMessage.case) @@ -269,37 +249,33 @@ export class WriterTransport { } dbg.log('stream ended') - dispatch({ type: 'transport.ended' }) + if (!ac.signal.aborted) { + this.#disconnect() + } } catch (error) { if (ac.signal.aborted) { return } dbg.log('stream error: %O', error) - dispatch({ type: 'transport.error', error }) + this.#disconnect(error) } })() + } - this.#streamAC = ac - this.#streamInput = input - this.#streamTask = task - this.#tokenPending = false // fresh stream — the previous token, if any, is gone + #disconnect(error?: unknown): void { + this.#closeStream() + this.#events.push({ + type: 'transport.stream.disconnected', + ...(error !== undefined && { error }), + }) } - async #closeStream(): Promise { + #closeStream(reason?: unknown): void { let ac = this.#streamAC let input = this.#streamInput - let task = this.#streamTask - this.#streamAC = null this.#streamInput = null - this.#streamTask = null - - if (!ac) { - return - } - - ac.abort(new Error('Stream disposed')) - input?.close() - await task + ac?.abort(reason ?? new Error('Stream disposed')) + input?.destroy() } } diff --git a/packages/topic/src/writer/types.ts b/packages/topic/src/writer/types.ts index c9b0634d..f7734956 100644 --- a/packages/topic/src/writer/types.ts +++ b/packages/topic/src/writer/types.ts @@ -1,5 +1,8 @@ import type { CompressionCodec } from '../codec.js' import type { TX } from '../tx.js' +import type { TopicWriter } from './writer.js' + +export type TopicTxWriter = Pick // Status of a single message acknowledged by the server. export type AckStatus = 'written' | 'skipped' | 'writtenInTx' @@ -44,12 +47,12 @@ export type TopicWriterOptions = { // Route writes by message group (mutually exclusive with partitionId). messageGroupId?: string - // Hard cap on the un-acknowledged bytes held in memory; write() throws when a - // message would exceed it. Default 256MiB. + // Byte limit for unacknowledged payloads after compression. + // write() throws when the budget would be exceeded. Default 256MiB. maxBufferBytes?: bigint // Cap the number of un-acknowledged (in-flight) messages. Default 1000. maxInflightCount?: number - // Background flush cadence in ms — bounds how long a small batch waits. Default 1000. + // Maximum background wait for a partial batch in ms. Default 1000. flushIntervalMs?: number // How often to refresh the auth token on the stream. Default 60s. diff --git a/packages/topic/src/writer/writer-runtime.ts b/packages/topic/src/writer/writer-runtime.ts index a4bcc675..69b2b246 100644 --- a/packages/topic/src/writer/writer-runtime.ts +++ b/packages/topic/src/writer/writer-runtime.ts @@ -8,8 +8,7 @@ import type { Driver } from '@ydbjs/core' import { loggers } from '@ydbjs/debug' import { type MachineRuntime, createMachineRuntime } from '@ydbjs/fsm' -import { type InitParams, WriterTransport } from './transport.js' -import type { TransportOutput } from './transport-state.js' +import { type InitParams, type TransportOutput, WriterTransport } from './transport.js' import type { TopicWriterOptions } from './types.js' import { type GlobalTimerName, @@ -20,6 +19,7 @@ import { type WriterOutput, type WriterState, createWriterCtx, + releaseState, writerTransition, } from './writer-state.js' @@ -40,19 +40,13 @@ let BACKOFF_MAX_MS = 30_000 type WriterEnv = { // I/O transport: WriterTransport - codec: number txIdentity?: { id: string; session: string } // timer durations (ms) - startTimeoutMs: number flushIntervalMs: number updateTokenIntervalMs: number gracefulShutdownTimeoutMs: number - // lifecycle - ac: AbortController - closedDeferred: PromiseWithResolvers - isFinalized: boolean // Keyed by GlobalTimerName — string-keyed like the reader, whose keys also // carry a partition suffix (the writer has no partition timers). timers: Map> @@ -90,7 +84,7 @@ let backoffDelay = function backoffDelay(attempts: number): number { let delayFor = function delayFor(ctx: FullCtx, which: GlobalTimerName): number { switch (which) { case 'start_timeout': - return ctx.startTimeoutMs + return DEFAULT_START_TIMEOUT_MS case 'retry_backoff': return backoffDelay(ctx.attempts) case 'recovery_window': @@ -145,6 +139,15 @@ let mapTransportOutput = function mapTransportOutput(output: TransportOutput): W } } +let finalize = function finalize(ctx: FullCtx, reason: unknown): void { + for (let handle of ctx.timers.values()) { + clearTimeout(handle) + } + ctx.timers.clear() + releaseState(ctx) + ctx.transport.destroy(reason) +} + // Builds the writer FSM and binds its effects to real I/O: transport connect/send, // timers with equal-jitter backoff, and transport-output → writer-event mapping. // The returned machine is the only handle the facade drives; every side effect @@ -158,26 +161,21 @@ export function createWriterRuntime(driver: Driver, options: TopicWriterOptions) } let transport = new WriterTransport(driver, initParams) + let codec = options.codec?.codec ?? Codec.RAW let env: WriterEnv = { // I/O transport, - codec: options.codec?.codec ?? Codec.RAW, ...(options.tx && { txIdentity: { id: options.tx.transactionId, session: options.tx.sessionId }, }), // timer durations (ms) - startTimeoutMs: DEFAULT_START_TIMEOUT_MS, flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, updateTokenIntervalMs: options.updateTokenIntervalMs ?? DEFAULT_UPDATE_TOKEN_INTERVAL_MS, gracefulShutdownTimeoutMs: options.gracefulShutdownTimeoutMs ?? DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_MS, - // lifecycle - ac: new AbortController(), - closedDeferred: Promise.withResolvers(), - isFinalized: false, timers: new Map(), } @@ -189,7 +187,7 @@ export function createWriterRuntime(driver: Driver, options: TopicWriterOptions) { retryOnSchemeError: options.retryOnSchemeError ?? false, recoveryWindowMs: options.recoveryWindowMs ?? DEFAULT_RECOVERY_WINDOW_MS, - codec: options.codec?.codec ?? Codec.RAW, + codec, } ) @@ -219,10 +217,6 @@ export function createWriterRuntime(driver: Driver, options: TopicWriterOptions) fullCtx.transport.connect(effect.getLastSeqNo) }, - 'writer.effect.transport.close': (fullCtx) => { - fullCtx.transport.close() - }, - 'writer.effect.send.write_request': (fullCtx, effect) => { evdbg.log( 'send write_request: %d messages (seqNo %s..%s)', @@ -255,7 +249,7 @@ export function createWriterRuntime(driver: Driver, options: TopicWriterOptions) clearTimerByKey(fullCtx, which) - let repeating = which === 'flush_tick' || which === 'update_token' + let repeating = which === 'update_token' || which === 'flush_tick' let delay = delayFor(fullCtx, which) let event = timerEvent(which) @@ -274,24 +268,19 @@ export function createWriterRuntime(driver: Driver, options: TopicWriterOptions) clearTimerByKey(fullCtx, effect.which) }, - 'writer.effect.finalize': (fullCtx, effect) => { - if (fullCtx.isFinalized) { - return - } - fullCtx.isFinalized = true - - for (let handle of fullCtx.timers.values()) { - clearTimeout(handle) - } - fullCtx.timers.clear() - - fullCtx.transport.destroy(effect.reason) - fullCtx.ac.abort(effect.reason) - fullCtx.closedDeferred.resolve() - }, + 'writer.effect.finalize': (fullCtx, effect) => finalize(fullCtx, effect.reason), }, }) + // Runtime faults bypass transition effects; resource ownership still ends here. + machine.signal.addEventListener( + 'abort', + () => finalize(ctx as FullCtx, machine.signal.reason), + { + once: true, + } + ) + // Route transport lifecycle events into the writer FSM. machine.ingest(transport.events, mapTransportOutput) diff --git a/packages/topic/src/writer/writer-state.test.ts b/packages/topic/src/writer/writer-state.test.ts index 8d676263..af0449ab 100644 --- a/packages/topic/src/writer/writer-state.test.ts +++ b/packages/topic/src/writer/writer-state.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest' +import { isRetryableTopicError } from '../retry.ts' import type { TransitionRuntime } from '@ydbjs/fsm' import { YDBError } from '@ydbjs/error' import { StatusIds_StatusCode } from '@ydbjs/api/operation' @@ -14,7 +15,6 @@ import { type WriterOutput, type WriterState, createWriterCtx, - isRetryableWriterError, writerTransition, } from './writer-state.ts' @@ -24,11 +24,20 @@ let limits: WriterLimits = { } let ctxWith = function ctxWith(overrides: Partial = {}): WriterCtx { - return { ...createWriterCtx(limits), ...overrides } + let ctx = { ...createWriterCtx(limits), ...overrides } + ctx.unsentBytes = ctx.messages + .slice(ctx.inflightCount) + .reduce((sum, message) => sum + BigInt(message.data.length), 0n) + return ctx } let msg = function msg(byte: number, seqNo = 0n): BufferedMessage { - return { data: new Uint8Array([byte]), uncompressedSize: 1n, seqNo, createdAt: new Date(0) } + return { + data: new Uint8Array([byte]), + uncompressedSize: 1n, + seqNo, + createdAt: new Date(0), + } } type Driven = { @@ -64,6 +73,34 @@ let effectTypes = function effectTypes(effects: WriterEffect[]): string[] { return effects.map((effect) => effect.type) } +test('checks batch readiness with linear work while small writes accumulate', () => { + let count = 1000 + let ctx = ctxWith({ limits: { ...limits, maxInflightCount: count } }) + let sizeReads = 0 + let sent: WriterEffect[] = [] + for (let i = 0; i < count; i++) { + let message = msg(i) + let payload = message.data + Object.defineProperty(message, 'data', { + get() { + sizeReads++ + return payload + }, + }) + drive('ready', { type: 'writer.write', message }, ctx) + let result = drive('ready', { type: 'writer.pump' }, ctx) + sent.push(...result.effects) + expect(result.effects).toHaveLength(Number(i === count - 1)) + } + expect(sent).toHaveLength(1) + expect(sent[0]).toMatchObject({ + type: 'writer.effect.send.write_request', + messages: { length: count }, + }) + expect(ctx.inflightCount).toBe(count) + expect(sizeReads).toBeLessThanOrEqual(count * 3) +}) + // Drive to ready with two unacked in-flight messages (seqNos 1, 2). let readyWithInflight = function readyWithInflight(): WriterCtx { let ctx = ctxWith() @@ -75,33 +112,31 @@ let readyWithInflight = function readyWithInflight(): WriterCtx { { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 0n }, ctx ) - drive('ready', { type: 'writer.pump' }, ctx) - expect(ctx.inflightLength).toBe(2) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + expect(ctx.inflightCount).toBe(2) return ctx } // ── error classification ──────────────────────────────────────────────────────── test('classifies a missing error as retryable', () => { - expect(isRetryableWriterError(undefined)).toBe(true) + expect(isRetryableTopicError(undefined)).toBe(true) }) test('classifies UNAVAILABLE as retryable', () => { - expect(isRetryableWriterError(new YDBError(StatusIds_StatusCode.UNAVAILABLE, []))).toBe(true) + expect(isRetryableTopicError(new YDBError(StatusIds_StatusCode.UNAVAILABLE, []))).toBe(true) }) test('classifies conditionally-retryable SESSION_EXPIRED as retryable for idempotent writes', () => { - expect(isRetryableWriterError(new YDBError(StatusIds_StatusCode.SESSION_EXPIRED, []))).toBe( - true - ) + expect(isRetryableTopicError(new YDBError(StatusIds_StatusCode.SESSION_EXPIRED, []))).toBe(true) }) test('classifies SCHEME_ERROR as fatal', () => { - expect(isRetryableWriterError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []))).toBe(false) + expect(isRetryableTopicError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []))).toBe(false) }) test('classifies SCHEME_ERROR as retryable when retryOnSchemeError is set', () => { - expect(isRetryableWriterError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []), true)).toBe( + expect(isRetryableTopicError(new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []), true)).toBe( true ) }) @@ -115,7 +150,7 @@ test('classifies a server frame-cap rejection as fatal', () => { Status.RESOURCE_EXHAUSTED, 'Received message larger than max (66060326 vs. 64000000)' ) - expect(isRetryableWriterError(error)).toBe(false) + expect(isRetryableTopicError(error)).toBe(false) }) test('classifies a client send-cap rejection as fatal', () => { @@ -124,7 +159,7 @@ test('classifies a client send-cap rejection as fatal', () => { Status.RESOURCE_EXHAUSTED, 'Attempted to send message with a size larger than 67108864' ) - expect(isRetryableWriterError(error)).toBe(false) + expect(isRetryableTopicError(error)).toBe(false) }) test('classifies a throttling RESOURCE_EXHAUSTED as retryable', () => { @@ -134,11 +169,11 @@ test('classifies a throttling RESOURCE_EXHAUSTED as retryable', () => { Status.RESOURCE_EXHAUSTED, 'Too many pending requests' ) - expect(isRetryableWriterError(error)).toBe(true) + expect(isRetryableTopicError(error)).toBe(true) }) test('classifies an unknown error as fatal', () => { - expect(isRetryableWriterError(new Error('message is larger than max allowed'))).toBe(false) + expect(isRetryableTopicError(new Error('message is larger than max allowed'))).toBe(false) }) // ── idle ──────────────────────────────────────────────────────────────────────── @@ -157,22 +192,21 @@ test('buffers a write received before connecting', () => { let d = drive('idle', { type: 'writer.write', message: msg(1) }, ctx) expect(d.state).toBe('idle') - expect(d.ctx.bufferLength).toBe(1) - expect(d.ctx.seqNoMode).toBe('auto') + expect(d.ctx.messages.length - d.ctx.inflightCount).toBe(1) + expect(d.ctx.messages[0]?.seqNo).toBe(0n) }) -test('records manual mode from a seqNo-carrying write', () => { +test('retains the user sequence number while buffering', () => { let ctx = ctxWith() drive('idle', { type: 'writer.write', message: msg(1, 5n) }, ctx) - expect(ctx.seqNoMode).toBe('manual') expect(ctx.lastSeqNo).toBe(5n) }) // ── connecting → ready (init + seqno recovery) ────────────────────────────────── test('recovers the server high-water mark on first init in auto mode', () => { - let ctx = ctxWith({ seqNoMode: 'auto' }) + let ctx = ctxWith() let d = drive( 'connecting', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 42n }, @@ -188,7 +222,8 @@ test('recovers the server high-water mark on first init in auto mode', () => { }) test('does not overwrite user seqNo with server value in manual mode', () => { - let ctx = ctxWith({ seqNoMode: 'manual', lastSeqNo: 5n }) + let ctx = ctxWith() + drive('connecting', { type: 'writer.write', message: msg(1, 5n) }, ctx) let d = drive( 'connecting', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 100n }, @@ -212,13 +247,13 @@ test('resets the retry counter on successful init', () => { // ── auto seqNo assigned at send time ───────────────────────────────────────────── test('assigns auto seqNos sequentially at send time', () => { - let ctx = ctxWith({ seqNoMode: 'auto', lastSeqNo: 10n, hasEverConnected: true }) + let ctx = ctxWith({ lastSeqNo: 10n, hasEverConnected: true }) // three buffered auto messages, unnumbered drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) drive('ready', { type: 'writer.write', message: msg(3) }, ctx) - let d = drive('ready', { type: 'writer.pump' }, ctx) + let d = drive('ready', { type: 'writer.timer.flush_tick' }, ctx) let batch = d.effects.find((e) => e.type === 'writer.effect.send.write_request') expect( @@ -226,37 +261,36 @@ test('assigns auto seqNos sequentially at send time', () => { batch.type === 'writer.effect.send.write_request' && batch.messages.map((m) => m.seqNo) ).toEqual([11n, 12n, 13n]) - expect(d.ctx.inflightLength).toBe(3) - expect(d.ctx.bufferLength).toBe(0) + expect(d.ctx.inflightCount).toBe(3) + expect(d.ctx.messages.length - d.ctx.inflightCount).toBe(0) expect(d.ctx.lastSeqNo).toBe(13n) }) test('does not send when the inflight window is full', () => { let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, limits: { ...limits, maxInflightCount: 1 }, }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) - let first = drive('ready', { type: 'writer.pump' }, ctx) - expect(first.ctx.inflightLength).toBe(1) + let first = drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + expect(first.ctx.inflightCount).toBe(1) - let second = drive('ready', { type: 'writer.pump' }, ctx) + let second = drive('ready', { type: 'writer.timer.flush_tick' }, ctx) expect( second.effects.find((e) => e.type === 'writer.effect.send.write_request') ).toBeUndefined() - expect(second.ctx.bufferLength).toBe(1) + expect(second.ctx.messages.length - second.ctx.inflightCount).toBe(1) }) // ── acknowledgements ───────────────────────────────────────────────────────────── test('moves acknowledged messages out of the inflight window', () => { - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true, lastSeqNo: 0n }) + let ctx = ctxWith({ hasEverConnected: true, lastSeqNo: 0n }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) let d = drive( 'ready', @@ -270,7 +304,7 @@ test('moves acknowledged messages out of the inflight window', () => { ctx ) - expect(d.ctx.inflightLength).toBe(0) + expect(d.ctx.inflightCount).toBe(0) let acks = d.emitted.find((o) => o.type === 'writer.acknowledgments') expect(acks && acks.type === 'writer.acknowledgments' && acks.acknowledgments.get(1n)).toBe( 'written' @@ -282,11 +316,11 @@ test('moves acknowledged messages out of the inflight window', () => { test('acknowledges only the contiguous prefix on a non-prefix ack set', () => { // Defensive: if the head is unacked, tail acks must not leave the window or fire // callbacks early — nothing is emitted until the head is acked. - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true, lastSeqNo: 0n }) + let ctx = ctxWith({ hasEverConnected: true, lastSeqNo: 0n }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) drive('ready', { type: 'writer.write', message: msg(3) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) let d = drive( 'ready', @@ -300,7 +334,7 @@ test('acknowledges only the contiguous prefix on a non-prefix ack set', () => { ctx ) - expect(d.ctx.inflightLength).toBe(3) + expect(d.ctx.inflightCount).toBe(3) expect(d.emitted.find((o) => o.type === 'writer.acknowledgments')).toBeUndefined() }) @@ -308,18 +342,18 @@ test('acknowledges only the contiguous prefix on a non-prefix ack set', () => { test('flushes immediately when nothing is pending', () => { let ctx = ctxWith({ hasEverConnected: true, lastSeqNo: 7n }) - let d = drive('ready', { type: 'writer.flush' }, ctx) + let d = drive('ready', { type: 'writer.flush', requestId: 1 }, ctx) let flushed = d.emitted.find((o) => o.type === 'writer.flushed') - expect(flushed).toEqual({ type: 'writer.flushed', lastSeqNo: 7n }) + expect(flushed).toEqual({ type: 'writer.flushed', requestId: 1, lastSeqNo: 7n }) }) test('emits flushed once the buffer drains after a flush request', () => { - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true }) + let ctx = ctxWith({ hasEverConnected: true }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) - drive('ready', { type: 'writer.flush' }, ctx) - expect(ctx.flushRequested).toBe(true) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + drive('ready', { type: 'writer.flush', requestId: 1 }, ctx) + expect([...ctx.flushes.values()]).toEqual([1]) let d = drive( 'ready', @@ -328,7 +362,7 @@ test('emits flushed once the buffer drains after a flush request', () => { ) expect(d.emitted.find((o) => o.type === 'writer.flushed')).toBeDefined() - expect(d.ctx.flushRequested).toBe(false) + expect(d.ctx.flushes.size).toBe(0) }) // ── reconnect ──────────────────────────────────────────────────────────────────── @@ -402,12 +436,12 @@ test('resolves a pending flush when a reconnect init drains the window via dedup // whose init covers all inflight messages drains the window via dedup with NO // write_response. A pending flush must be resolved from the init path — else it // hangs forever, stalling any tx commit awaiting it. - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true, lastSeqNo: 0n }) + let ctx = ctxWith({ hasEverConnected: true, lastSeqNo: 0n }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) - drive('ready', { type: 'writer.flush' }, ctx) - expect(ctx.flushRequested).toBe(true) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + drive('ready', { type: 'writer.flush', requestId: 1 }, ctx) + expect([...ctx.flushes.values()]).toEqual([1]) let d = drive( 'connecting', @@ -417,9 +451,10 @@ test('resolves a pending flush when a reconnect init drains the window via dedup expect(d.emitted.find((o) => o.type === 'writer.flushed')).toEqual({ type: 'writer.flushed', + requestId: 1, lastSeqNo: 2n, }) - expect(d.ctx.flushRequested).toBe(false) + expect(d.ctx.flushes.size).toBe(0) }) test('fails terminally when the recovery window expires', () => { @@ -439,11 +474,11 @@ test('fails terminally when the recovery window expires during a connect attempt test('resends unacked inflight and drops server-acked on reconnect init', () => { // two messages sent (seqNo 1,2); server persisted 1 only. - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true, lastSeqNo: 0n }) + let ctx = ctxWith({ hasEverConnected: true, lastSeqNo: 0n }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) - expect(ctx.inflightLength).toBe(2) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + expect(ctx.inflightCount).toBe(2) let d = drive( 'connecting', @@ -452,8 +487,8 @@ test('resends unacked inflight and drops server-acked on reconnect init', () => ) // seqNo 1 dropped (already written), seqNo 2 rewound into the buffer for resend. - expect(d.ctx.inflightLength).toBe(0) - expect(d.ctx.bufferLength).toBe(1) + expect(d.ctx.inflightCount).toBe(0) + expect(d.ctx.messages.length - d.ctx.inflightCount).toBe(1) let recovered = d.emitted.find((o) => o.type === 'writer.acknowledgments') expect( recovered && @@ -474,19 +509,19 @@ test('closes immediately when nothing is pending', () => { expect(d.state).toBe('closed') expect(d.emitted.find((o) => o.type === 'writer.closed')).toBeDefined() - expect(effectTypes(d.effects)).toContain('writer.effect.transport.close') + expect(effectTypes(d.effects)).toEqual(['writer.effect.finalize']) }) test('drains before closing when messages are pending', () => { - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true }) + let ctx = ctxWith({ hasEverConnected: true }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) let d = drive('ready', { type: 'writer.close' }, ctx) - expect(d.state).toBe('closing') + expect(d.state).toBe('ready') let done = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 1n, status: 'written' }] }, d.ctx ) @@ -532,13 +567,12 @@ test('close during reconnect clears the stale recovery_window timer', () => { // close must cancel it so it cannot cut the graceful drain short — close is bounded by // graceful_timeout, not recoveryWindowMs (mirrors the reader). let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, - bufferLength: 1, messages: [msg(1)], }) let d = drive('reconnecting', { type: 'writer.close' }, ctx) - expect(d.state).toBe('closing') + expect(d.state).toBe('reconnecting') + expect(ctx.closeRequested).toBe(true) expect(d.effects).toContainEqual({ type: 'writer.effect.timer.clear', which: 'recovery_window', @@ -547,12 +581,11 @@ test('close during reconnect clears the stale recovery_window timer', () => { test('fails when the graceful timeout fires with undelivered messages', () => { let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, - bufferLength: 1, + closeRequested: true, messages: [msg(1)], }) - let d = drive('closing', { type: 'writer.timer.graceful_timeout' }, ctx) + let d = drive('ready', { type: 'writer.timer.graceful_timeout' }, ctx) // Undelivered messages on a forced shutdown is a flush failure, not a clean close. expect(d.state).toBe('errored') @@ -560,8 +593,8 @@ test('fails when the graceful timeout fires with undelivered messages', () => { }) test('closes cleanly when the graceful timeout fires with nothing pending', () => { - let ctx = ctxWith({ hasEverConnected: true }) - let d = drive('closing', { type: 'writer.timer.graceful_timeout' }, ctx) + let ctx = ctxWith({ hasEverConnected: true, closeRequested: true }) + let d = drive('ready', { type: 'writer.timer.graceful_timeout' }, ctx) expect(d.state).toBe('closed') }) @@ -571,8 +604,7 @@ test('destroys from any state, tearing down the transport', () => { let d = drive('ready', { type: 'writer.destroy', reason: new Error('boom') }, ctx) expect(d.state).toBe('closed') - expect(effectTypes(d.effects)).toContain('writer.effect.transport.close') - expect(effectTypes(d.effects)).toContain('writer.effect.finalize') + expect(effectTypes(d.effects)).toEqual(['writer.effect.finalize']) }) test('ignores events once closed', () => { @@ -580,7 +612,7 @@ test('ignores events once closed', () => { let d = drive('closed', { type: 'writer.write', message: msg(1) }, ctx) expect(d.state).toBe('closed') - expect(d.ctx.bufferLength).toBe(0) + expect(d.ctx.messages.length - d.ctx.inflightCount).toBe(0) }) // ── closing drain: init gate, acks, flush ─────────────────────────────────────── @@ -591,10 +623,10 @@ test('emits acknowledgments for acks landing during the closing drain', () => { // the facade decrements the budget before writer.closed resets it (review WRITER-2). let ctx = readyWithInflight() let d = drive('ready', { type: 'writer.close' }, ctx) - expect(d.state).toBe('closing') + expect(d.state).toBe('ready') let partial = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 1n, status: 'written' }] }, ctx ) @@ -603,10 +635,10 @@ test('emits acknowledgments for acks landing during the closing drain', () => { expect(acks && acks.type === 'writer.acknowledgments' && acks.acknowledgments.get(1n)).toBe( 'written' ) - expect(partial.state).toBe('closing') + expect(partial.state).toBe('ready') let final = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 2n, status: 'written' }] }, ctx ) @@ -623,11 +655,11 @@ test('orders recovered acks before flushed before closed on a mid-close reconnec // on `flushed` before `closed` rejects the remainder (review WRITER-2 / WRITER-3). let ctx = readyWithInflight() drive('ready', { type: 'writer.close' }, ctx) - drive('closing', { type: 'writer.flush' }, ctx) - drive('closing', { type: 'writer.stream.disconnected' }, ctx) + drive('ready', { type: 'writer.flush', requestId: 1 }, ctx) + drive('ready', { type: 'writer.stream.disconnected' }, ctx) let d = drive( - 'closing', + 'reconnecting', { type: 'writer.stream.init_response', sessionId: 's2', lastSeqNo: 2n }, ctx ) @@ -655,11 +687,11 @@ test('resolves a flush issued during closing when the ack drains the window', () let ctx = readyWithInflight() drive('ready', { type: 'writer.close' }, ctx) - let f = drive('closing', { type: 'writer.flush' }, ctx) + let f = drive('ready', { type: 'writer.flush', requestId: 1 }, ctx) expect(f.emitted.filter((o) => o.type === 'writer.flushed')).toHaveLength(0) let d = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [ @@ -678,19 +710,17 @@ test('emits acknowledgments, flushed, closed in that order on the final closing // The exact emission order on the write_response drain path is load-bearing for // the facade (budget → flush waiter → terminal), see review WRITER-2 / WRITER-3. let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, + closeRequested: true, lastSeqNo: 2n, - flushRequested: true, messages: [msg(1, 1n), msg(2, 2n)], - bufferStart: 2, - bufferLength: 0, - inflightStart: 0, - inflightLength: 2, + inflightCount: 2, }) + ctx.flushes.set(ctx.messages.at(-1)!, 1) + let d = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [ @@ -712,18 +742,15 @@ test('emits acknowledgments, flushed, closed in that order on the final closing test('does not emit flushed on a clean closing drain when no flush was requested', () => { // Regression guard around the closing flush handler: no spurious writer.flushed. let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, + closeRequested: true, lastSeqNo: 1n, messages: [msg(1, 1n)], - bufferStart: 1, - bufferLength: 0, - inflightStart: 0, - inflightLength: 1, + inflightCount: 1, }) let d = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 1n, status: 'written' }] }, ctx ) @@ -737,21 +764,18 @@ test('reports each seqNo exactly once when a mid-close init dedups part of the w // rewound, resent, and acked by a write_response. Every seqNo must be reported // exactly once across all acknowledgment emissions — no double count of freed bytes. let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, + closeRequested: true, lastSeqNo: 2n, messages: [msg(1, 1n), msg(2, 2n)], - bufferStart: 2, - bufferLength: 0, - inflightStart: 0, - inflightLength: 2, + inflightCount: 2, }) let seen: bigint[] = [] let freed = 0n let init = drive( - 'closing', + 'connecting', { type: 'writer.stream.init_response', sessionId: 's2', lastSeqNo: 1n }, ctx ) @@ -761,14 +785,14 @@ test('reports each seqNo exactly once when a mid-close init dedups part of the w freed += out.freedBytes } } - expect(init.state).toBe('closing') + expect(init.state).toBe('ready') // seqNo 2 was rewound into the buffer — resend it - let pumped = drive('closing', { type: 'writer.pump' }, ctx) + let pumped = drive('ready', { type: 'writer.pump' }, ctx) expect(pumped.effects.some((e) => e.type === 'writer.effect.send.write_request')).toBe(true) let acked = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 2n, status: 'written' }] }, ctx ) @@ -789,10 +813,10 @@ test('keeps pumping the close drain when the seqNo mark was already recovered', let ctx = readyWithInflight() drive('ready', { type: 'writer.write', message: msg(3) }, ctx) let d = drive('ready', { type: 'writer.close' }, ctx) - expect(d.state).toBe('closing') + expect(d.state).toBe('ready') expect(d.dispatched).toContainEqual({ type: 'writer.pump' }) - let pumped = drive('closing', { type: 'writer.pump' }, ctx) + let pumped = drive('ready', { type: 'writer.pump' }, ctx) let batch = pumped.effects.find((e) => e.type === 'writer.effect.send.write_request') expect(batch).toBeDefined() // Auto seqNo continues from the recovered mark, never from 0. @@ -810,26 +834,26 @@ test('suppresses the close drain until the first init recovers the mark', () => drive('idle', { type: 'writer.write', message: msg(1) }, ctx) drive('idle', { type: 'writer.start' }, ctx) let d = drive('connecting', { type: 'writer.close' }, ctx) - expect(d.state).toBe('closing') + expect(d.state).toBe('connecting') // The pump is gated — nothing is sent with a fabricated seqNo. - let gated = drive('closing', { type: 'writer.pump' }, ctx) + let gated = drive('connecting', { type: 'writer.pump' }, ctx) expect(gated.effects).toEqual([]) expect(ctx.messages[0]!.seqNo).toBe(0n) // The init lands mid-close: the server already has 41 messages. The never-sent // buffered message must not be reported as acknowledged/deduplicated locally… let init = drive( - 'closing', + 'connecting', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 41n }, ctx ) - expect(init.state).toBe('closing') + expect(init.state).toBe('ready') expect(init.emitted.filter((o) => o.type === 'writer.acknowledgments')).toHaveLength(0) expect(init.dispatched).toContainEqual({ type: 'writer.pump' }) // …and the drain resumes numbering from the recovered mark. - let pumped = drive('closing', { type: 'writer.pump' }, ctx) + let pumped = drive('ready', { type: 'writer.pump' }, ctx) let batch = pumped.effects.find((e) => e.type === 'writer.effect.send.write_request') expect(batch).toBeDefined() expect( @@ -846,26 +870,25 @@ test('holds manual-seqNo messages until first init and sends them unrenumbered', drive('idle', { type: 'writer.start' }, ctx) drive('connecting', { type: 'writer.write', message: msg(1, 7n) }, ctx) drive('connecting', { type: 'writer.write', message: msg(2, 9n) }, ctx) - expect(ctx.seqNoMode).toBe('manual') let closing = drive('connecting', { type: 'writer.close' }, ctx) - expect(closing.state).toBe('closing') + expect(closing.state).toBe('connecting') // pump before init: gated, nothing on the wire - let gated = drive('closing', { type: 'writer.pump' }, ctx) + let gated = drive('connecting', { type: 'writer.pump' }, ctx) expect(gated.effects).toHaveLength(0) - expect(ctx.inflightLength).toBe(0) + expect(ctx.inflightCount).toBe(0) // init arrives (server has never seen this producer) — the drain resumes let init = drive( - 'closing', + 'connecting', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 0n }, ctx ) - expect(init.state).toBe('closing') + expect(init.state).toBe('ready') expect(init.dispatched).toContainEqual({ type: 'writer.pump' }) - let pumped = drive('closing', { type: 'writer.pump' }, ctx) + let pumped = drive('ready', { type: 'writer.pump' }, ctx) let batch = pumped.effects.find((e) => e.type === 'writer.effect.send.write_request') expect(batch).toBeDefined() expect( @@ -876,7 +899,7 @@ test('holds manual-seqNo messages until first init and sends them unrenumbered', // the gate never stalls the drain forever: the final acks complete the close let done = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [ @@ -898,22 +921,22 @@ test('does not locally dedup never-sent manual messages on a close-racing init', drive('idle', { type: 'writer.start' }, ctx) drive('connecting', { type: 'writer.write', message: msg(1, 5n) }, ctx) drive('connecting', { type: 'writer.close' }, ctx) - drive('closing', { type: 'writer.pump' }, ctx) + drive('connecting', { type: 'writer.pump' }, ctx) // server already has lastSeqNo=100 — greater than the buffered seqNo 5 let init = drive( - 'closing', + 'connecting', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 100n }, ctx ) // never-sent messages must not be reported as acknowledged/skipped locally expect(init.emitted.filter((o) => o.type === 'writer.acknowledgments')).toHaveLength(0) - expect(ctx.bufferLength).toBe(1) - expect(init.state).toBe('closing') + expect(ctx.messages.length - ctx.inflightCount).toBe(1) + expect(init.state).toBe('ready') // and the drain proceeds with the user's number - let pumped = drive('closing', { type: 'writer.pump' }, ctx) + let pumped = drive('ready', { type: 'writer.pump' }, ctx) expect(pumped.effects.find((e) => e.type === 'writer.effect.send.write_request')).toBeDefined() }) @@ -932,8 +955,8 @@ test('errors the gated close when the first init never arrives', () => { ) ).toBe(true) - drive('closing', { type: 'writer.pump' }, ctx) - let timedOut = drive('closing', { type: 'writer.timer.graceful_timeout' }, ctx) + drive('connecting', { type: 'writer.pump' }, ctx) + let timedOut = drive('connecting', { type: 'writer.timer.graceful_timeout' }, ctx) expect(timedOut.state).toBe('errored') let types = timedOut.emitted.map((o) => o.type) @@ -950,19 +973,23 @@ test('defers a flush requested during a gated close until the init unlocks the d drive('connecting', { type: 'writer.write', message: msg(1) }, ctx) drive('connecting', { type: 'writer.close' }, ctx) - let flushed = drive('closing', { type: 'writer.flush' }, ctx) + let flushed = drive('connecting', { type: 'writer.flush', requestId: 1 }, ctx) expect(flushed.emitted).toHaveLength(0) // not drained — must not resolve now - expect(ctx.flushRequested).toBe(true) + expect([...ctx.flushes.values()]).toEqual([1]) // the pump requestFlush dispatched is still gated - let gated = drive('closing', { type: 'writer.pump' }, ctx) + let gated = drive('connecting', { type: 'writer.pump' }, ctx) expect(gated.effects).toHaveLength(0) // init → pump → ack: flush resolves inside the drain - drive('closing', { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 0n }, ctx) - drive('closing', { type: 'writer.pump' }, ctx) + drive( + 'connecting', + { type: 'writer.stream.init_response', sessionId: 's1', lastSeqNo: 0n }, + ctx + ) + drive('ready', { type: 'writer.pump' }, ctx) let acked = drive( - 'closing', + 'ready', { type: 'writer.stream.write_response', acks: [{ seqNo: 1n, status: 'written' }] }, ctx ) @@ -976,13 +1003,13 @@ test('defers a flush requested during a gated close until the init unlocks the d // ── resource / memory ──────────────────────────────────────────────────────────── test('reclaims the message array as acknowledgments accumulate', () => { - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true }) + let ctx = ctxWith({ hasEverConnected: true }) // Write, send and acknowledge many messages one at a time. Garbage compaction // must keep the backing array bounded — a leak would grow it without limit. for (let i = 0; i < 5000; i++) { drive('ready', { type: 'writer.write', message: msg(1) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) drive( 'ready', { @@ -997,28 +1024,75 @@ test('reclaims the message array as acknowledgments accumulate', () => { }) test('releases the message buffer on destroy', () => { - let ctx = ctxWith({ seqNoMode: 'auto', hasEverConnected: true }) + let ctx = ctxWith({ hasEverConnected: true }) drive('ready', { type: 'writer.write', message: msg(1) }, ctx) drive('ready', { type: 'writer.write', message: msg(2) }, ctx) - drive('ready', { type: 'writer.pump' }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) let d = drive('ready', { type: 'writer.destroy', reason: new Error('x') }, ctx) expect(d.state).toBe('closed') expect(d.ctx.messages).toHaveLength(0) - expect(d.ctx.bufferLength).toBe(0) - expect(d.ctx.inflightLength).toBe(0) + expect(d.ctx.messages.length - d.ctx.inflightCount).toBe(0) + expect(d.ctx.inflightCount).toBe(0) }) test('releases the message buffer on a terminal error', () => { let ctx = ctxWith({ - seqNoMode: 'auto', hasEverConnected: true, messages: [msg(1)], - bufferLength: 1, }) let d = drive('reconnecting', { type: 'writer.timer.recovery_window' }, ctx) expect(d.state).toBe('errored') expect(d.ctx.messages).toHaveLength(0) }) + +test('releases recovered payloads after every reconnect without normal acknowledgments', () => { + let ctx = ctxWith({ hasEverConnected: true }) + for (let i = 1; i <= 10; i++) { + drive('ready', { type: 'writer.write', message: msg(i) }, ctx) + drive('ready', { type: 'writer.timer.flush_tick' }, ctx) + let recovered = drive( + 'connecting', + { type: 'writer.stream.init_response', sessionId: `s${i}`, lastSeqNo: BigInt(i) }, + ctx + ) + expect(recovered.emitted).toContainEqual({ + type: 'writer.acknowledgments', + acknowledgments: new Map([[BigInt(i), 'skipped']]), + freedBytes: 1n, + }) + expect(ctx.messages).toHaveLength(0) + expect(ctx.messages.length - ctx.inflightCount).toBe(0) + expect(ctx.inflightCount).toBe(0) + } +}) + +test('fills a 48 MiB payload batch without charging metadata or framing', () => { + let ctx = ctxWith() + let payload = new Uint8Array(24 * 1024 * 1024) + for (let i = 0; i < 2; i++) { + drive( + 'ready', + { + type: 'writer.write', + message: { + data: payload, + uncompressedSize: BigInt(payload.length), + seqNo: 0n, + createdAt: new Date(0), + metadataItems: { trace: new Uint8Array(256) }, + }, + }, + ctx + ) + } + let result = drive('ready', { type: 'writer.pump' }, ctx) + expect(result.effects).toHaveLength(1) + expect(result.effects[0]).toMatchObject({ + type: 'writer.effect.send.write_request', + messages: { length: 2 }, + }) + expect(ctx.unsentBytes).toBe(0n) +}) diff --git a/packages/topic/src/writer/writer-state.ts b/packages/topic/src/writer/writer-state.ts index 0f268d7c..dd935750 100644 --- a/packages/topic/src/writer/writer-state.ts +++ b/packages/topic/src/writer/writer-state.ts @@ -1,15 +1,13 @@ import { create } from '@bufbuild/protobuf' import { timestampFromDate } from '@bufbuild/protobuf/wkt' -import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { type StreamWriteMessage_WriteRequest_MessageData, StreamWriteMessage_WriteRequest_MessageDataSchema, } from '@ydbjs/api/topic' import { loggers } from '@ydbjs/debug' -import { YDBError } from '@ydbjs/error' import type { TransitionResult, TransitionRuntime } from '@ydbjs/fsm' -import { isRetryableError, isRetryableStreamError } from '@ydbjs/retry' -import { ClientError, Status } from 'nice-grpc' + +import { isRetryableTopicError } from '../retry.js' import type { AckStatus, WriteAck } from './types.js' @@ -17,28 +15,21 @@ import type { AckStatus, WriteAck } from './types.js' // with no I/O. Everything here mutates `ctx` in place and returns the next // state + a list of effects for the runtime to execute — see writer-runtime.ts // for the I/O side. The buffer is a sliding window (see WriterCtx) and byte -// budgeting lives in the facade, so the transition only counts messages. +// admission budgeting lives in the facade; the transition owns batching. // // The full transition map (table + diagram) lives in packages/topic/ARCHITECTURE.md — // update it in the same commit when you change this dispatch. let dbg = loggers.topic.extend('writer') -// Hard service limits (bytes). -export const MAX_BATCH_BYTES = 48n * 1024n * 1024n // one WriteRequest frame stays under 48MiB -export const MAX_PAYLOAD_BYTES = 48n * 1024n * 1024n // single message payload cap +// Payload limits leave headroom for protocol framing. +export const MAX_BATCH_BYTES = 48n * 1024n * 1024n // compressed-payload batch target +export const MAX_PAYLOAD_BYTES = 48n * 1024n * 1024n // single uncompressed payload cap // ── State / context ───────────────────────────────────────────────────────────── // `closed` = graceful/destroyed terminal; `errored` = fatal terminal. Both are final. -export type WriterState = - | 'idle' - | 'connecting' - | 'ready' - | 'reconnecting' - | 'closing' - | 'closed' - | 'errored' +export type WriterState = 'idle' | 'connecting' | 'ready' | 'reconnecting' | 'closed' | 'errored' // A message living in the sliding window before/while it is on the wire. // In auto mode `seqNo` stays 0n until the message is actually sent (assigned in `pump`); @@ -60,17 +51,13 @@ export type WriterLimits = { } // Pure logical context — mutated synchronously inside the transition only. -// The single message array is a sliding window: [garbage | inflight | buffer]. -// garbage = [0, inflightStart) (acked, awaiting compaction) -// inflight = [inflightStart, bufferStart) (sent, awaiting ack) -// buffer = [bufferStart, messages.length) (not yet sent) +// Messages retain only unacknowledged writes. The prefix [0, inflightCount) was +// sent on the current stream; the remaining suffix is waiting to be sent. export type WriterCtx = { - // connection identity - sessionId: string + // Initial seqNo recovery hasEverConnected: boolean // seqNo bookkeeping - seqNoMode: 'auto' | 'manual' | null lastSeqNo: bigint // reconnect bookkeeping @@ -83,8 +70,11 @@ export type WriterCtx = { // transition owns whether to arm the `recovery_window` timer based on this. recoveryWindowMs: number - // flush barrier - flushRequested: boolean + // Each flush captures its last accepted message; calls at the same boundary share one completion. + flushes: Map + closeRequested: boolean + // A partial batch is due after its timer fires or after an interrupted stream. + batchDue: boolean // The session codec (Codec enum value / custom id) — validated against // InitResponse.supported_codecs: a disallowed codec is fatal at init, before any @@ -92,19 +82,15 @@ export type WriterCtx = { // with an opaque BAD_REQUEST at the first WriteRequest). codec: number - // sliding-window buffer (see the diagram above) messages: BufferedMessage[] - bufferStart: number - bufferLength: number - inflightStart: number - inflightLength: number + inflightCount: number + // Sum of compressed payload bytes in the unsent suffix; avoids rescanning a partial batch on every write. + unsentBytes: bigint limits: WriterLimits } -// Timer names shared with the reader, plus the writer-only flush cadence. The -// writer has no partition-scoped timers, so a TimerRef is just the global name — -// the reader's TimerRef adds a partition-keyed variant. +// Writer lifecycle timers are scoped to the complete write session. export type GlobalTimerName = | 'start_timeout' | 'retry_backoff' @@ -119,7 +105,7 @@ export type WriterEvent = // user (dispatched by the facade) | { type: 'writer.start' } | { type: 'writer.write'; message: BufferedMessage } - | { type: 'writer.flush' } + | { type: 'writer.flush'; requestId: number } | { type: 'writer.close' } | { type: 'writer.destroy'; reason?: unknown } // internal self-dispatch — fsm has no `always`/`after`, so the send loop is an explicit event @@ -147,7 +133,6 @@ export type WriterEvent = // transport.* = socket lifecycle only; send.* = anything written to the stream. export type WriterEffect = | { type: 'writer.effect.transport.connect'; getLastSeqNo: boolean } - | { type: 'writer.effect.transport.close' } | { type: 'writer.effect.send.write_request' messages: StreamWriteMessage_WriteRequest_MessageData[] @@ -159,14 +144,13 @@ export type WriterEffect = export type WriterOutput = | { type: 'writer.session'; sessionId: string; lastSeqNo: bigint; nextSeqNo: bigint } - // `freedBytes` = compressed bytes that left the window with this ack batch, so - // the facade can decrement its byte budget without re-tracking message sizes. + // Compressed payload bytes reclaimed by this ACK batch. | { type: 'writer.acknowledgments' acknowledgments: Map freedBytes: bigint } - | { type: 'writer.flushed'; lastSeqNo: bigint } + | { type: 'writer.flushed'; requestId: number; lastSeqNo: bigint } | { type: 'writer.reconnecting'; attempt: number; error?: unknown } | { type: 'writer.error'; error: unknown } | { type: 'writer.closed'; reason?: unknown } @@ -180,10 +164,8 @@ export let createWriterCtx = function createWriterCtx( options?: { retryOnSchemeError?: boolean; recoveryWindowMs?: number; codec?: number } ): WriterCtx { return { - sessionId: '', hasEverConnected: false, - seqNoMode: null, lastSeqNo: 0n, attempts: 0, @@ -191,102 +173,63 @@ export let createWriterCtx = function createWriterCtx( retryOnSchemeError: options?.retryOnSchemeError ?? false, recoveryWindowMs: options?.recoveryWindowMs ?? Infinity, - flushRequested: false, + flushes: new Map(), + closeRequested: false, + batchDue: false, // 1 = Codec.RAW, the writer default. codec: options?.codec ?? 1, messages: [], - bufferStart: 0, - bufferLength: 0, - inflightStart: 0, - inflightLength: 0, + inflightCount: 0, + unsentBytes: 0n, limits, } } -// A stream error is retryable when the writer should reconnect transparently. -// Topic writes are idempotent (dedup by producerId+seqNo), so we use the -// idempotent classification — unlike the plain stream classifier, this retries -// the "conditionally" YDB statuses (SESSION_EXPIRED, UNDETERMINED, TIMEOUT). -// A clean stream end with no error object is also retryable (server-side reconnect). -// SCHEME_ERROR is fatal unless `retryOnSchemeError` is set (wait for topic creation). -export let isRetryableWriterError = function isRetryableWriterError( - error: unknown, - retryOnSchemeError = false -): boolean { - if (error === undefined || error === null) { - return true - } - - if (isPayloadTooLargeError(error)) { - return false - } - - if ( - retryOnSchemeError && - error instanceof YDBError && - error.code === StatusIds_StatusCode.SCHEME_ERROR - ) { - return true - } - - return isRetryableStreamError(error) || isRetryableError(error, true) -} - -// A size-limit rejection is deterministic — resending the same oversized frame can -// only fail again, so it must be fatal (Go demotes this case explicitly). Every -// size rejection observed against a real server (tests/writer-protocol.test.ts) is -// a gRPC ClientError RESOURCE_EXHAUSTED whose details carry a size complaint: -// server frame cap: 'Received message larger than max (66060326 vs. 64000000)' -// client send cap: 'Attempted to send message with a size larger than 67108864' -// (grpc-js receive paths use the same 'larger than' wording). The code alone is not -// enough — RESOURCE_EXHAUSTED also covers genuine throttling, which SHOULD be -// retried — so the details text narrows it. Everything else the server could send -// (e.g. a YDBError BAD_REQUEST issue) is already non-retryable via the generic -// classifier and needs no special case here. -let isPayloadTooLargeError = function isPayloadTooLargeError(error: unknown): boolean { - return ( - error instanceof ClientError && - error.code === Status.RESOURCE_EXHAUSTED && - /larger than/i.test(error.details) - ) -} - let allDrained = function allDrained(ctx: WriterCtx): boolean { - return ctx.bufferLength === 0 && ctx.inflightLength === 0 + return ctx.messages.length === 0 } // The window has work and headroom: something is buffered and inflight has room. let canSend = function canSend(ctx: WriterCtx): boolean { - return ctx.bufferLength > 0 && ctx.inflightLength < ctx.limits.maxInflightCount + return ( + ctx.inflightCount < ctx.messages.length && ctx.inflightCount < ctx.limits.maxInflightCount + ) } -// Resolve a pending flush the moment the window is empty. Every path that can -// drain the buffer (a write_response ack, or a reconnect whose init dedups all -// in-flight messages) must call this — otherwise a flush that drains via the -// dedup path never emits writer.flushed and the caller hangs forever. -let resolveFlushIfDrained = function resolveFlushIfDrained( +let requestFlush = function requestFlush( ctx: WriterCtx, - runtime: WriterRuntime + runtime: WriterRuntime, + requestId: number ): void { - if (ctx.flushRequested && allDrained(ctx)) { - ctx.flushRequested = false - runtime.emit({ type: 'writer.flushed', lastSeqNo: ctx.lastSeqNo }) + let last = ctx.messages.at(-1) + if (!last) { + runtime.emit({ type: 'writer.flushed', requestId, lastSeqNo: ctx.lastSeqNo }) + return } + + ctx.flushes.set(last, requestId) + runtime.dispatch({ type: 'writer.pump' }) } -// Record a flush request. Honored in every live state — a flush issued while the -// writer is still connecting must resolve once messages drain after init, not be -// dropped. Resolves immediately when there is nothing pending. -let requestFlush = function requestFlush(ctx: WriterCtx, runtime: WriterRuntime): void { - ctx.flushRequested = true - resolveFlushIfDrained(ctx, runtime) - // Still pending — kick the send loop to drain it. - if (ctx.flushRequested) { - runtime.dispatch({ type: 'writer.pump' }) +// Auto sequence numbers are assigned at send time, so a flush follows message identity until ACK. +let removeAcknowledged = function removeAcknowledged( + ctx: WriterCtx, + count: number +): WriterOutput[] { + let completed: WriterOutput[] = [] + for (let message of ctx.messages.splice(0, count)) { + let requestId = ctx.flushes.get(message) + if (requestId !== undefined) { + ctx.flushes.delete(message) + completed.push({ type: 'writer.flushed', requestId, lastSeqNo: message.seqNo }) + } } + ctx.inflightCount -= count + + return completed } // One stream attempt: open the transport and arm its watchdog. Shared by every @@ -324,36 +267,46 @@ let toMessageData = function toMessageData( // batch-byte limits, assigning auto seqNos as we go. Mutates the window in place // (buffer → inflight) and returns the on-wire messages. Synchronous by design. let formBatch = function formBatch(ctx: WriterCtx): StreamWriteMessage_WriteRequest_MessageData[] { - let batch: StreamWriteMessage_WriteRequest_MessageData[] = [] - let batchBytes = 0n - let end = ctx.bufferStart + ctx.bufferLength - - for (let i = ctx.bufferStart; i < end; i++) { - let message = ctx.messages[i]! - let size = BigInt(message.data.length) + let available = ctx.limits.maxInflightCount - ctx.inflightCount + if ( + !ctx.batchDue && + !ctx.closeRequested && + ctx.flushes.size === 0 && + ctx.messages.length - ctx.inflightCount < available && + ctx.unsentBytes < ctx.limits.maxBatchBytes + ) { + return [] + } - if (batch.length > 0 && batchBytes + size > ctx.limits.maxBatchBytes) { + let count = 0 + let batchBytes = 0n + for (let i = ctx.inflightCount; i < ctx.messages.length; i++) { + let size = BigInt(ctx.messages[i]!.data.length) + if (count > 0 && batchBytes + size > ctx.limits.maxBatchBytes) { break } - - if (ctx.inflightLength + batch.length >= ctx.limits.maxInflightCount) { + count++ + batchBytes += size + if (count === available || batchBytes === ctx.limits.maxBatchBytes) { break } + } - // Auto mode: the seqNo is assigned now, at send time, from the high-water mark. + let batch: StreamWriteMessage_WriteRequest_MessageData[] = [] + for (let i = ctx.inflightCount; i < ctx.inflightCount + count; i++) { + let message = ctx.messages[i]! if (message.seqNo === 0n) { ctx.lastSeqNo += 1n message.seqNo = ctx.lastSeqNo } - batch.push(toMessageData(message)) - batchBytes += size } - let count = batch.length - ctx.bufferStart += count - ctx.bufferLength -= count - ctx.inflightLength += count + ctx.inflightCount += count + ctx.unsentBytes -= batchBytes + if (ctx.inflightCount === ctx.messages.length) { + ctx.batchDue = false + } return batch } @@ -361,33 +314,34 @@ let formBatch = function formBatch(ctx: WriterCtx): StreamWriteMessage_WriteRequ // Apply a server init: recover the seqNo high-water mark once (auto numbering), // then drop any server-persisted in-flight messages and rewind the rest for resend. // -// The dedup runs on EVERY init, including reconnects: YDB reports last_seq_no even -// when get_last_seq_no is false (proven in tests/writer-protocol.test.ts), so we -// skip resending messages the server already has — like the Java SDK. We only -// request get_last_seq_no on the first connect (like Go) to avoid its cost. If a -// reconnect ever reported 0, dropAckedAndRewind drops nothing and we resend -// everything; the server dedups by producerId+seqNo — correct either way, just -// less efficient. So this is an optimization, not a correctness dependency. +// YDB reports last_seq_no on reconnect even when get_last_seq_no is false. Only +// previously sent messages may be removed using that watermark; unsent auto messages +// have no sequence number yet. If the server reports zero, replay remains safe through +// producer+seqNo deduplication. let applyInit = function applyInit( ctx: WriterCtx, sessionId: string, serverLastSeqNo: bigint, runtime: WriterRuntime ): void { - ctx.sessionId = sessionId - if (!ctx.hasEverConnected) { - // Trust the server's high-water mark exactly once. Manual mode keeps the - // user's numbers; auto mode continues above the recovered value. - if (ctx.seqNoMode !== 'manual' && serverLastSeqNo > ctx.lastSeqNo) { + // No message can be sent before the first init, so a nonzero buffered seqNo + // is user-provided. Automatic numbering continues above the recovered value. + let manual = (ctx.messages[0]?.seqNo ?? 0n) !== 0n + if (!manual && serverLastSeqNo > ctx.lastSeqNo) { ctx.lastSeqNo = serverLastSeqNo } + ctx.hasEverConnected = true } - let { recovered, freedBytes } = dropAckedAndRewind(ctx, serverLastSeqNo) + let { recovered, freedBytes, flushed } = dropAckedAndRewind(ctx, serverLastSeqNo) if (recovered.size > 0) { - runtime.emit({ type: 'writer.acknowledgments', acknowledgments: recovered, freedBytes }) + runtime.emit({ + type: 'writer.acknowledgments', + acknowledgments: recovered, + freedBytes, + }) } runtime.emit({ @@ -396,6 +350,9 @@ let applyInit = function applyInit( lastSeqNo: ctx.lastSeqNo, nextSeqNo: ctx.lastSeqNo + 1n, }) + for (let output of flushed) { + runtime.emit(output) + } } // Drop in-flight messages the server already persisted (seqNo <= serverLastSeqNo), @@ -405,15 +362,15 @@ let applyInit = function applyInit( let dropAckedAndRewind = function dropAckedAndRewind( ctx: WriterCtx, serverLastSeqNo: bigint -): { recovered: Map; freedBytes: bigint } { +): { recovered: Map; freedBytes: bigint; flushed: WriterOutput[] } { let recovered = new Map() let freedBytes = 0n // In-flight seqNos are strictly increasing (assigned in order in formBatch), so // `seqNo <= serverLastSeqNo` splits the in-flight range at one boundary — walk // the acked prefix, exactly like acknowledge() walks the acked prefix. - let inflightEnd = ctx.bufferStart - let i = ctx.inflightStart + let inflightEnd = ctx.inflightCount + let i = 0 while (i < inflightEnd) { let message = ctx.messages[i]! if (message.seqNo === 0n || message.seqNo > serverLastSeqNo) { @@ -424,15 +381,19 @@ let dropAckedAndRewind = function dropAckedAndRewind( i += 1 } - ctx.bufferStart = i - ctx.bufferLength = ctx.messages.length - i - ctx.inflightStart = i - ctx.inflightLength = 0 + let resend = i < ctx.inflightCount + for (let j = i; j < inflightEnd; j++) { + ctx.unsentBytes += BigInt(ctx.messages[j]!.data.length) + } + + let flushed = removeAcknowledged(ctx, i) + ctx.inflightCount = 0 + ctx.batchDue = ctx.messages.length > 0 && (ctx.batchDue || resend) - return { recovered, freedBytes } + return { recovered, freedBytes, flushed } } -// Move server-acknowledged messages out of the in-flight window into garbage. +// Remove server-acknowledged messages from the in-flight prefix. // The server acks the in-flight prefix in order, so we walk from the head and // stop at the first unacked message. We report only the messages actually removed // from the window, so the emitted acks and the freed-byte total can never drift @@ -440,7 +401,7 @@ let dropAckedAndRewind = function dropAckedAndRewind( let acknowledge = function acknowledge( ctx: WriterCtx, acks: WriteAck[] -): { acknowledgments: Map; freedBytes: bigint } { +): { acknowledgments: Map; freedBytes: bigint; flushed: WriterOutput[] } { let status = new Map() for (let ack of acks) { status.set(ack.seqNo, ack.status) @@ -448,9 +409,9 @@ let acknowledge = function acknowledge( let acknowledgments = new Map() let freedBytes = 0n - let inflightEnd = ctx.bufferStart - while (ctx.inflightStart < inflightEnd) { - let message = ctx.messages[ctx.inflightStart]! + let count = 0 + while (count < ctx.inflightCount) { + let message = ctx.messages[count]! let messageStatus = status.get(message.seqNo) if (messageStatus === undefined) { break @@ -458,25 +419,12 @@ let acknowledge = function acknowledge( acknowledgments.set(message.seqNo, messageStatus) freedBytes += BigInt(message.data.length) - ctx.inflightStart += 1 - ctx.inflightLength -= 1 + count += 1 } - compactGarbage(ctx) + let flushed = removeAcknowledged(ctx, count) - return { acknowledgments, freedBytes } -} - -// Reclaim the garbage prefix by splicing it out and rebasing the window pointers. -let compactGarbage = function compactGarbage(ctx: WriterCtx): void { - let garbageLength = ctx.inflightStart - if (garbageLength === 0) { - return - } - - ctx.messages.splice(0, garbageLength) - ctx.inflightStart = 0 - ctx.bufferStart -= garbageLength + return { acknowledgments, freedBytes, flushed } } // Append a message to the buffer. Total by design — seqNo-mode validation @@ -487,17 +435,13 @@ let compactGarbage = function compactGarbage(ctx: WriterCtx): void { let enqueue = function enqueue(ctx: WriterCtx, message: BufferedMessage): void { let providedSeqNo = message.seqNo !== 0n - if (ctx.seqNoMode === null) { - ctx.seqNoMode = providedSeqNo ? 'manual' : 'auto' - } - // Manual mode: lastSeqNo tracks the user's high-water mark for resend/recovery. - if (providedSeqNo && message.seqNo > ctx.lastSeqNo) { + if (providedSeqNo) { ctx.lastSeqNo = message.seqNo } ctx.messages.push(message) - ctx.bufferLength += 1 + ctx.unsentBytes += BigInt(message.data.length) } // ── Terminal / transitions ────────────────────────────────────────────────────── @@ -514,6 +458,7 @@ let terminate = function terminate( ctx.lastError = reason runtime.emit({ type: 'writer.error', error: reason }) } + runtime.emit({ type: 'writer.closed', reason }) // Drop any still-buffered/in-flight messages so their payloads can be GC'd — @@ -526,7 +471,6 @@ let terminate = function terminate( // buffered lifecycle outputs (writer.closed / writer.error) are delivered first. final: { reason }, effects: [ - { type: 'writer.effect.transport.close' }, // No per-timer clears — the finalize handler clears the whole timer map. { type: 'writer.effect.finalize', reason }, ], @@ -534,15 +478,15 @@ let terminate = function terminate( } // Free the message window. Called on terminal stop to release payload memory. -let releaseState = function releaseState(ctx: WriterCtx): void { +export let releaseState = function releaseState(ctx: WriterCtx): void { ctx.messages = [] - ctx.bufferStart = 0 - ctx.bufferLength = 0 - ctx.inflightStart = 0 - ctx.inflightLength = 0 + ctx.inflightCount = 0 + ctx.unsentBytes = 0n + ctx.batchDue = false + ctx.flushes.clear() } -// ready + (write/pump/flush_tick): drain buffer → inflight, one batch per event. +// Drain buffered messages into the in-flight prefix, one batch per event. let pump = function pump( ctx: WriterCtx, runtime: WriterRuntime @@ -575,9 +519,11 @@ let codecRejectedByTopic = function codecRejectedByTopic( if (!supportedCodecs || supportedCodecs.length === 0) { return undefined } + if (supportedCodecs.includes(ctx.codec)) { return undefined } + return new Error( `Codec ${ctx.codec} is not allowed by the topic (supported codecs: ${supportedCodecs.join(', ')})` ) @@ -595,9 +541,13 @@ let toReady = function toReady( if (codecError) { return terminate(ctx, 'errored', codecError, runtime) } + ctx.attempts = 0 applyInit(ctx, event.sessionId, event.lastSeqNo, runtime) - resolveFlushIfDrained(ctx, runtime) + let closed = finishDrain(ctx, runtime) + if (closed) { + return closed + } runtime.dispatch({ type: 'writer.pump' }) @@ -621,13 +571,15 @@ let toReconnecting = function toReconnecting( if (error !== undefined) { ctx.lastError = error } + // Reconnecting must not restart the batching delay for already accepted messages. + ctx.batchDue = ctx.messages.length > 0 + runtime.emit({ type: 'writer.reconnecting', attempt: ctx.attempts, ...(error !== undefined && { error }), }) - // No transport.close here — the transport already closed its own stream on - // disconnect, and the reconnect happens via transport.connect (which reopens). + // The disconnected transport already released its stream; connect reopens it. let effects: WriterEffect[] = [ { type: 'writer.effect.timer.clear', which: 'start_timeout' }, { type: 'writer.effect.timer.clear', which: 'flush_tick' }, @@ -637,49 +589,22 @@ let toReconnecting = function toReconnecting( // Arm the terminal deadline only when recovery is bounded. Unbounded (Infinity) // means reconnect forever — the transition owns that policy so the emitted effects // reflect it (model-testable), instead of the runtime silently dropping the timer. - if (Number.isFinite(ctx.recoveryWindowMs)) { + if (!ctx.closeRequested && Number.isFinite(ctx.recoveryWindowMs)) { effects.push({ type: 'writer.effect.timer.schedule', which: 'recovery_window' }) } + return { state: 'reconnecting', effects } } -// Closing drain gate: finalize once the window is empty, otherwise keep pumping. -let closeWhenDrained = function closeWhenDrained( +let finishDrain = function finishDrain( ctx: WriterCtx, runtime: WriterRuntime -): TransitionResult | void { - if (allDrained(ctx)) { +): TransitionResult | undefined { + if (ctx.closeRequested && allDrained(ctx)) { return terminate(ctx, 'closed', new Error('Writer closed'), runtime) } - runtime.dispatch({ type: 'writer.pump' }) -} -// Enter graceful shutdown. If nothing is pending, finalize now; otherwise drain -// the buffer (over the live stream, or over a reconnect if one is already -// scheduled) bounded by the graceful-shutdown timeout. Retry/recovery timers are -// intentionally preserved so a close issued while reconnecting still flushes. -let toClosing = function toClosing( - ctx: WriterCtx, - runtime: WriterRuntime -): TransitionResult { - let drained = closeWhenDrained(ctx, runtime) - if (drained) { - return drained - } - - return { - state: 'closing', - effects: [ - // Closing may be entered while reconnecting — cancel a stale recovery_window - // so it cannot cut the graceful drain short (close is bounded by - // graceful_timeout, like the reader). start_timeout / retry_backoff are left - // armed: the closing state uses them to keep reconnecting to finish the drain. - { type: 'writer.effect.timer.clear', which: 'recovery_window' }, - { type: 'writer.effect.timer.clear', which: 'flush_tick' }, - { type: 'writer.effect.timer.clear', which: 'update_token' }, - { type: 'writer.effect.timer.schedule', which: 'graceful_timeout' }, - ], - } + return undefined } // ── Transition ────────────────────────────────────────────────────────────────── @@ -696,171 +621,138 @@ export let writerTransition = function writerTransition( runtime: WriterRuntime ): TransitionResult | void { let state = runtime.state - - // Global: hard destroy from any non-terminal state. - if (state !== 'closed' && state !== 'errored' && event.type === 'writer.destroy') { - return terminate(ctx, 'closed', event.reason ?? new Error('Writer destroyed'), runtime) + if (state === 'closed' || state === 'errored') { + return ignored(state, event) } - switch (state) { - case 'idle': { - switch (event.type) { - case 'writer.start': - return { state: 'connecting', effects: connectEffects(ctx) } - - case 'writer.write': - enqueue(ctx, event.message) - return - - case 'writer.close': - return terminate( - ctx, - 'closed', - new Error('Writer closed before start'), - runtime - ) - - default: - return ignored(state, event) + switch (event.type) { + case 'writer.destroy': + return terminate(ctx, 'closed', event.reason ?? new Error('Writer destroyed'), runtime) + case 'writer.close': { + if (ctx.closeRequested) { + return } - } - - case 'connecting': { - switch (event.type) { - case 'writer.write': - enqueue(ctx, event.message) - return - - case 'writer.flush': - requestFlush(ctx, runtime) - return - case 'writer.stream.init_response': - return toReady(ctx, event, runtime) - - case 'writer.timer.start_timeout': - return toReconnecting(ctx, undefined, runtime) - - case 'writer.stream.disconnected': - if (!isRetryableWriterError(event.error, ctx.retryOnSchemeError)) { - return terminate(ctx, 'errored', event.error, runtime) - } - return toReconnecting(ctx, event.error, runtime) + ctx.closeRequested = true + let closed = finishDrain(ctx, runtime) + if (closed) { + return closed + } - // The recovery window is armed while reconnecting and can elapse during a - // connect attempt — without this the terminal bound would never fire. - case 'writer.timer.recovery_window': - return terminate( - ctx, - 'errored', - ctx.lastError ?? new Error('Writer recovery window expired'), - runtime - ) + if (state === 'idle') { + runtime.dispatch({ type: 'writer.start' }) + } - case 'writer.close': - return toClosing(ctx, runtime) + if (state === 'ready') { + runtime.dispatch({ type: 'writer.pump' }) + } - default: - return ignored(state, event) + return { + effects: [ + { type: 'writer.effect.timer.clear', which: 'recovery_window' }, + { type: 'writer.effect.timer.schedule', which: 'graceful_timeout' }, + ], } } + case 'writer.write': + if (ctx.closeRequested) { + return ignored(state, event) + } - case 'ready': { - switch (event.type) { - case 'writer.write': - enqueue(ctx, event.message) - runtime.dispatch({ type: 'writer.pump' }) - return - - case 'writer.pump': - case 'writer.timer.flush_tick': - return pump(ctx, runtime) - - case 'writer.stream.write_response': { - let { acknowledgments, freedBytes } = acknowledge(ctx, event.acks) - if (acknowledgments.size > 0) { - runtime.emit({ - type: 'writer.acknowledgments', - acknowledgments, - freedBytes, - }) - } - resolveFlushIfDrained(ctx, runtime) - runtime.dispatch({ type: 'writer.pump' }) - return - } - - case 'writer.flush': - requestFlush(ctx, runtime) - return - - case 'writer.timer.update_token': - return { effects: [{ type: 'writer.effect.send.update_token' }] } + enqueue(ctx, event.message) + if (state === 'ready') { + runtime.dispatch({ type: 'writer.pump' }) + } + return + case 'writer.flush': + requestFlush(ctx, runtime, event.requestId) + return + case 'writer.timer.flush_tick': + if (ctx.messages.length > ctx.inflightCount) { + ctx.batchDue = true + } - case 'writer.stream.token_response': - return + if (state === 'ready') { + return pump(ctx, runtime) + } + return + case 'writer.timer.graceful_timeout': + if (!ctx.closeRequested) { + return ignored(state, event) + } - case 'writer.stream.disconnected': - if (!isRetryableWriterError(event.error, ctx.retryOnSchemeError)) { - return terminate(ctx, 'errored', event.error, runtime) - } - return toReconnecting(ctx, event.error, runtime) + if (!allDrained(ctx)) { + return terminate( + ctx, + 'errored', + new Error('Graceful shutdown timed out with undelivered messages'), + runtime + ) + } - case 'writer.close': - return toClosing(ctx, runtime) + return finishDrain(ctx, runtime) + } - default: - return ignored(state, event) + switch (state) { + case 'idle': + if (event.type === 'writer.start') { + return { state: 'connecting', effects: connectEffects(ctx) } } - } - - case 'reconnecting': { - switch (event.type) { - case 'writer.write': - enqueue(ctx, event.message) - return - case 'writer.flush': - requestFlush(ctx, runtime) - return + return ignored(state, event) - // A connect attempt whose init lands here (start_timeout fired just before - // the init was dequeued, so the stream is still open) is a live session — - // honor it rather than dropping it and forcing a wasted reconnect. + case 'connecting': + case 'reconnecting': + switch (event.type) { case 'writer.stream.init_response': return toReady(ctx, event, runtime) + case 'writer.timer.start_timeout': + if (state === 'connecting') { + return toReconnecting(ctx, undefined, runtime) + } + return ignored(state, event) case 'writer.timer.retry_backoff': + if (state !== 'reconnecting') { + return ignored(state, event) + } + ctx.attempts += 1 return { state: 'connecting', effects: connectEffects(ctx) } + case 'writer.stream.disconnected': + if (!isRetryableTopicError(event.error, ctx.retryOnSchemeError)) { + return terminate(ctx, 'errored', event.error, runtime) + } + + if (state === 'connecting') { + return toReconnecting(ctx, event.error, runtime) + } + if (event.error !== undefined) { + ctx.lastError = event.error + } + return case 'writer.timer.recovery_window': + if (ctx.closeRequested) { + return ignored(state, event) + } + return terminate( ctx, 'errored', ctx.lastError ?? new Error('Writer recovery window expired'), runtime ) - - case 'writer.stream.disconnected': - // Already backing off — record the reason but stay put. - if (event.error !== undefined) { - ctx.lastError = event.error - } - return - - case 'writer.close': - return toClosing(ctx, runtime) - default: return ignored(state, event) } - } - case 'closing': { + case 'ready': switch (event.type) { + case 'writer.pump': + return pump(ctx, runtime) case 'writer.stream.write_response': { - let { acknowledgments, freedBytes } = acknowledge(ctx, event.acks) + let { acknowledgments, freedBytes, flushed } = acknowledge(ctx, event.acks) if (acknowledgments.size > 0) { runtime.emit({ type: 'writer.acknowledgments', @@ -868,92 +760,31 @@ export let writerTransition = function writerTransition( freedBytes, }) } - resolveFlushIfDrained(ctx, runtime) - return closeWhenDrained(ctx, runtime) - } - case 'writer.pump': - case 'writer.timer.flush_tick': - // Never assign auto seqNos from an unrecovered high-water mark: a close() - // racing the first init would number from 0n and the server would dedup - // the whole batch as already-written — silent data loss on a clean close. - // The init_response handler below resumes the drain once the mark is known. - if (!ctx.hasEverConnected) { - return - } - return pump(ctx, runtime) - - case 'writer.flush': - requestFlush(ctx, runtime) - return - - // A reconnect completed mid-close — recover and keep draining. A codec the - // topic rejects makes the drain impossible: fail the close instead of - // letting the server kill the session at the first WriteRequest. - case 'writer.stream.init_response': { - let codecError = codecRejectedByTopic(ctx, event.supportedCodecs) - if (codecError) { - return terminate(ctx, 'errored', codecError, runtime) - } - applyInit(ctx, event.sessionId, event.lastSeqNo, runtime) - resolveFlushIfDrained(ctx, runtime) - return ( - closeWhenDrained(ctx, runtime) ?? { - effects: [ - { type: 'writer.effect.timer.clear', which: 'start_timeout' }, - ], - } - ) - } - - case 'writer.timer.retry_backoff': - return { effects: connectEffects(ctx) } - - case 'writer.timer.graceful_timeout': - // Forced shutdown with messages still pending is a failure to flush — - // surface it (as `errored`) so close() rejects instead of silently - // dropping undelivered writes (critical for tx commit integrity). - if (!allDrained(ctx)) { - return terminate( - ctx, - 'errored', - new Error('Graceful shutdown timed out with undelivered messages'), - runtime - ) + for (let output of flushed) { + runtime.emit(output) } - return closeWhenDrained(ctx, runtime) - case 'writer.timer.start_timeout': - // Retry the drain over a fresh stream (bounded by graceful_timeout). - return { - effects: [ - { type: 'writer.effect.timer.clear', which: 'start_timeout' }, - { type: 'writer.effect.timer.schedule', which: 'retry_backoff' }, - ], + let closed = finishDrain(ctx, runtime) + if (closed) { + return closed } + runtime.dispatch({ type: 'writer.pump' }) + return + } + case 'writer.timer.update_token': + return { effects: [{ type: 'writer.effect.send.update_token' }] } + case 'writer.stream.token_response': + return case 'writer.stream.disconnected': - // Retry the drain over a fresh stream (bounded by graceful_timeout); - // give up terminally on a fatal error. - if (!isRetryableWriterError(event.error, ctx.retryOnSchemeError)) { + if (!isRetryableTopicError(event.error, ctx.retryOnSchemeError)) { return terminate(ctx, 'errored', event.error, runtime) } - return { - effects: [ - { type: 'writer.effect.timer.clear', which: 'start_timeout' }, - { type: 'writer.effect.timer.schedule', which: 'retry_backoff' }, - ], - } - // New writes are rejected once closing (facade throws before dispatch), - // so ignore anything else. + return toReconnecting(ctx, event.error, runtime) default: return ignored(state, event) } - } - - case 'closed': - case 'errored': - return ignored(state, event) } } diff --git a/packages/topic/src/writer/writer.contract.test.ts b/packages/topic/src/writer/writer.contract.test.ts index c657ec55..a160bc11 100644 --- a/packages/topic/src/writer/writer.contract.test.ts +++ b/packages/topic/src/writer/writer.contract.test.ts @@ -4,9 +4,10 @@ import { getEventListeners } from 'node:events' import { Codec } from '@ydbjs/api/topic' import { StatusIds_StatusCode } from '@ydbjs/api/operation' import { YDBError } from '@ydbjs/error' -import { expect, test } from 'vitest' +import { expect, test, vi } from 'vitest' +import { ClientError, Status } from 'nice-grpc' -import { GZIP_CODEC } from '../codec.ts' +import { GZIP_CODEC, RAW_CODEC } from '../codec.ts' import type { TX } from '../tx.ts' import { failureResponse, @@ -16,6 +17,7 @@ import { tick, writeResponse, } from './writer.fixtures.ts' +import { WriterTransport } from './transport.ts' import { createTopicTxWriter, createTopicWriter } from './writer.ts' let bytes = function bytes(...values: number[]): Uint8Array { @@ -93,10 +95,9 @@ test('assigns auto seqNos at send time and acknowledges a flush', async () => { writer.write(bytes(1)) writer.write(bytes(2)) + let flushed = writer.flush() await settle() - // Eager per-write pumping may split these across batches — assert the seqNos - // were assigned sequentially at send time, in order, across all sent writes. let seqNos = stream.sent .filter((m) => m.clientMessage.case === 'writeRequest') .flatMap((m) => @@ -106,7 +107,6 @@ test('assigns auto seqNos at send time and acknowledges a flush', async () => { ) expect(seqNos).toEqual([1n, 2n]) - let flushed = writer.flush() stream.respond(writeResponse([{ seqNo: 1n }, { seqNo: 2n }])) await expect(flushed).resolves.toBe(2n) @@ -114,13 +114,19 @@ test('assigns auto seqNos at send time and acknowledges a flush', async () => { await writer.close() }) -test('generates a producer id when none is provided', async () => { +test('uses distinct producer identities for independently created default writers', async () => { let { driver, waitForNextStream } = makeFakeTopicDriver() - using _writer = createTopicWriter(driver, { topic: '/t' }) - - let stream = await waitForNextStream() - let init = await stream.waitForInit() - expect(init.producerId).toMatch(/^producer-/) + using first = createTopicWriter(driver, { topic: '/t' }) + using second = createTopicWriter(driver, { topic: '/t' }) + first.write(bytes(1)) + second.write(bytes(2)) + let firstStream = await waitForNextStream() + let secondStream = await waitForNextStream() + let firstInit = await firstStream.waitForInit() + let secondInit = await secondStream.waitForInit() + expect(firstInit.producerId).not.toBe('') + expect(secondInit.producerId).not.toBe('') + expect(secondInit.producerId).not.toBe(firstInit.producerId) }) test('continues auto seqNo from the recovered server high-water mark', async () => { @@ -182,7 +188,16 @@ test('resends unacked messages after a transparent reconnect', async () => { expect(resent.messages.map((m) => m.seqNo)).toEqual([2n]) }) -test('does not fail a pending flush across a retryable reconnect', async () => { +test.for([ + { name: 'clean end', error: undefined }, + { + name: 'server deadline', + error: new ClientError('/stream', Status.DEADLINE_EXCEEDED, 'deadline'), + }, + { name: 'server timeout', error: new YDBError(StatusIds_StatusCode.TIMEOUT, []) }, + { name: 'expired session', error: new YDBError(StatusIds_StatusCode.SESSION_EXPIRED, []) }, + { name: 'indeterminate result', error: new YDBError(StatusIds_StatusCode.UNDETERMINED, []) }, +])('preserves the pending flush across $name', async ({ error }, tc) => { let { driver, waitForNextStream } = makeFakeTopicDriver() using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) @@ -194,11 +209,17 @@ test('does not fail a pending flush across a retryable reconnect', async () => { writer.write(bytes(1)) await first.waitForWrite() - let flushed = writer.flush() + let flushed = writer.flush(tc.signal) let settledEarly = false void flushed.then(() => (settledEarly = true)).catch(() => (settledEarly = true)) - first.disconnect() + if (error instanceof YDBError) { + first.respond(failureResponse(error.code)) + } else if (error) { + first.fail(error) + } else { + first.disconnect() + } await settle() expect(settledEarly).toBe(false) @@ -333,7 +354,11 @@ test('rejects a non-increasing manual seqNo', async () => { test('throws synchronously when a write would exceed maxBufferBytes', async () => { let { driver, waitForNextStream } = makeFakeTopicDriver() - using writer = createTopicWriter(driver, { topic: '/t', producer: 'p', maxBufferBytes: 4n }) + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxBufferBytes: 4n, + }) let stream = await waitForNextStream() await stream.waitForInit() @@ -347,7 +372,11 @@ test('throws synchronously when a write would exceed maxBufferBytes', async () = test('reclaims buffer budget once acknowledgments free bytes', async () => { let { driver, waitForNextStream } = makeFakeTopicDriver() - using writer = createTopicWriter(driver, { topic: '/t', producer: 'p', maxBufferBytes: 3n }) + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxBufferBytes: 3n, + }) let stream = await waitForNextStream() await stream.waitForInit() @@ -399,7 +428,7 @@ test('publishes opened, session, and acknowledgment diagnostics events', async ( // acknowledged: per-status breakdown + acked bytes, not just a total count. expect(acks.payloads.reduce((sum, a) => sum + a.written, 0)).toBe(1) - expect(acks.payloads.reduce((sum, a) => sum + a.bytes, 0n)).toBeGreaterThan(0n) + expect(acks.payloads.reduce((sum, a) => sum + a.bytes, 0n)).toBe(1n) }) test('emits a flush tracing span', async () => { @@ -1408,3 +1437,686 @@ test('coalesces update-token requests until one is acknowledged', async () => { let tokenFrames = stream.sent.filter((m) => m.clientMessage.case === 'updateTokenRequest') expect(tokenFrames).toHaveLength(1) }) + +test('keeps a later flush pending until its preceding write is acknowledged', async () => { + // The fake controls ack delivery while both calls share the facade output queue. + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + + let first = writer.flush() + writer.write(bytes(1)) + let completed = false + let second = writer.flush().then((seqNo) => { + completed = true + return seqNo + }) + await expect(first).resolves.toBe(0n) + await settle() + expect(completed).toBe(false) + stream.respond(writeResponse([{ seqNo: 1n }])) + await expect(second).resolves.toBe(1n) +}) + +test('rejects an invalid date before accepting the write and remains usable', async () => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + expect(() => writer.write(bytes(1), { createdAt: new Date(NaN), seqNo: 10n })).toThrow( + /valid date/ + ) + writer.write(bytes(2)) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + let request = await stream.waitForWrite() + expect(request.messages[0]!.seqNo).toBe(1n) + let flushed = writer.flush() + stream.respond(writeResponse([{ seqNo: 1n }])) + await expect(flushed).resolves.toBe(1n) +}) + +test('cleans up the stream and timers when a runtime effect throws', async () => { + // A thrown effect is an internal fault; a real server cannot trigger it deterministically. + vi.useFakeTimers() + let failure = new Error('send effect failed') + let send = vi.spyOn(WriterTransport.prototype, 'sendBatch').mockImplementationOnce(() => { + throw failure + }) + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + expect(vi.getTimerCount()).toBeGreaterThan(0) + writer.write(bytes(1)) + await expect(writer.flush()).rejects.toBe(failure) + await settle() + expect(stream.wasAborted()).toBe(true) + expect(vi.getTimerCount()).toBe(0) + await expect(writer.close()).rejects.toBe(failure) + } finally { + send.mockRestore() + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('charges only compressed payload bytes and restores capacity after acknowledgment', async () => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p', maxBufferBytes: 1n }) + let metadataItems = { trace: new Uint8Array(100) } + writer.write(bytes(), { metadataItems }) + writer.write(bytes(1), { metadataItems }) + writer.write(bytes()) + expect(() => writer.write(bytes(2))).toThrow(/buffer is full/) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + let flushed = writer.flush() + let request = await stream.waitForWrite() + expect(request.messages.map((message) => message.data.length)).toEqual([0, 1, 0]) + stream.respond(writeResponse([{ seqNo: 1n }, { seqNo: 2n }, { seqNo: 3n }])) + await flushed + expect(() => writer.write(bytes(3))).not.toThrow() +}) + +test('restores the payload budget when reconnect initialization acknowledges a message', async () => { + // The fake drops the response after persistence, then reports the recovered seqNo. + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p', maxBufferBytes: 1n }) + let metadataItems = { trace: new Uint8Array(100) } + let first = await waitForNextStream() + first.respond(initResponse(0n)) + writer.write(bytes(1), { metadataItems }) + await first.waitForWrite() + expect(() => writer.write(bytes(2), { metadataItems })).toThrow(/buffer is full/) + let flushed = writer.flush() + first.disconnect() + let second = await waitForNextStream() + second.respond(initResponse(1n)) + await expect(flushed).resolves.toBe(1n) + expect(() => writer.write(bytes(2), { metadataItems })).not.toThrow() +}) + +test('snapshots metadata and avoids retaining a larger backing buffer', async () => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + let backing = new Uint8Array(1024 * 1024) + let payload = backing.subarray(0, 3) + let metadataItems = { trace: bytes(7) } + writer.write(payload, { metadataItems }) + metadataItems.trace[0] = 9 + metadataItems.trace = new Uint8Array(1024) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + let message = (await stream.waitForWrite()).messages[0]! + expect(message.data.buffer.byteLength).toBe(3) + expect(message.metadataItems).toMatchObject([{ key: 'trace', value: bytes(7) }]) +}) + +test.each([0n, -1n, 1n << 63n])( + 'keeps auto numbering available after rejecting seqNo %s', + async (seqNo) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + expect(() => writer.write(bytes(1), { seqNo })).toThrow(/positive int64/) + writer.write(bytes(2)) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + expect((await stream.waitForWrite()).messages[0]!.seqNo).toBe(1n) + } +) + +test('retains falsy destruction reasons on close and flush', async () => { + await Promise.all( + [0, false, ''].map(async (reason) => { + let { driver } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + writer.write(bytes(1)) + writer.destroy(reason) + await expect(writer.flush()).rejects.toBe(reason) + await expect(writer.close()).rejects.toBe(reason) + }) + ) +}) + +test('sends full in-flight windows without waiting for the flush interval', async () => { + // Hold the clock still to distinguish protocol progress from timer-driven sends. + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxInflightCount: 1, + flushIntervalMs: 2_147_483_647, + }) + writer.write(bytes(1)) + writer.write(bytes(2)) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await stream.waitForWrite() + await settle() + let writes = () => + stream.sent.flatMap((frame) => + frame.clientMessage.case === 'writeRequest' + ? frame.clientMessage.value.messages + : [] + ) + expect(writes().map((message) => message.seqNo)).toEqual([1n]) + stream.respond(writeResponse([{ seqNo: 1n }])) + await settle() + expect(writes().map((message) => message.seqNo)).toEqual([1n, 2n]) + stream.respond(writeResponse([{ seqNo: 2n }])) + await settle() + writer.write(bytes(3)) + await settle() + expect(writes().map((message) => message.seqNo)).toEqual([1n, 2n, 3n]) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test.each(['before', 'after'])( + 'preserves manual flush results when the first write arrives %s init', + async (timing) => { + // Control init delivery and lose the first ack to exercise reconnect reconciliation. + let { driver, waitForNextStream } = makeFakeTopicDriver() + let acknowledged: Array<[bigint, string]> = [] + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + onAck: (seqNo, status) => acknowledged.push([seqNo, status]), + }) + let first = await waitForNextStream() + await first.waitForInit() + if (timing === 'before') { + writer.write(bytes(1), { seqNo: 1n }) + } + first.respond(initResponse(100n)) + await settle() + if (timing === 'after') { + writer.write(bytes(1), { seqNo: 1n }) + } + let request = await first.waitForWrite() + expect(request.messages.map((message) => message.seqNo)).toEqual([1n]) + let flushed = writer.flush() + first.disconnect() + let second = await waitForNextStream() + second.respond(initResponse(100n)) + await expect(flushed).resolves.toBe(1n) + expect(acknowledged).toEqual([[1n, 'skipped']]) + writer.write(bytes(2), { seqNo: 2n }) + let next = await second.waitForWrite() + expect(next.messages.map((message) => message.seqNo)).toEqual([2n]) + let nextFlush = writer.flush() + second.respond(writeResponse([{ seqNo: 2n, status: 'skipped' }])) + await expect(nextFlush).resolves.toBe(2n) + } +) + +test('cancels a stale reconnect backoff when a late init arrives during close', async () => { + // The server init must land between the connection watchdog and its retry timer. + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + gracefulShutdownTimeoutMs: 60_000, + }) + writer.write(bytes(1)) + let closed = writer.close() + void closed.catch(() => {}) + let stream = await waitForNextStream() + await stream.waitForInit() + await vi.advanceTimersByTimeAsync(30_000) + stream.respond(initResponse(0n)) + await stream.waitForWrite() + await vi.advanceTimersByTimeAsync(50) + expect(stream.wasAborted()).toBe(false) + stream.respond(writeResponse([{ seqNo: 1n }])) + await closed + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('waits for every reconnect init before sending a closing drain', async () => { + // Pause the replacement stream before init so closing cannot bypass its handshake. + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + let first = await waitForNextStream() + first.respond(initResponse(0n)) + writer.write(bytes(1)) + let flushed = writer.flush() + await first.waitForWrite() + first.respond(writeResponse([{ seqNo: 1n }])) + await flushed + first.disconnect() + await settle() + writer.write(bytes(2)) + await vi.advanceTimersByTimeAsync(50) + let second = await waitForNextStream() + await second.waitForInit() + let closed = writer.close() + void closed.catch(() => {}) + await settle() + expect( + second.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ).toHaveLength(0) + second.respond(initResponse(1n)) + let request = await second.waitForWrite() + expect(request.messages.map((message) => message.seqNo)).toEqual([2n]) + second.respond(writeResponse([{ seqNo: 2n }])) + await closed + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('flushes isolated messages on their interval even after a long idle period', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + let sent = () => + stream.sent.flatMap((frame) => + frame.clientMessage.case === 'writeRequest' + ? frame.clientMessage.value.messages.map((message) => message.seqNo) + : [] + ) + writer.write(bytes(1)) + await settle() + expect(sent()).toEqual([]) + await vi.advanceTimersByTimeAsync(999) + expect(sent()).toEqual([]) + await vi.advanceTimersByTimeAsync(1) + expect(sent()).toEqual([1n]) + stream.respond(writeResponse([{ seqNo: 1n }])) + await settle() + await vi.advanceTimersByTimeAsync(10 * 60 * 1000) + writer.write(bytes(2)) + await settle() + expect(sent()).toEqual([1n]) + await vi.advanceTimersByTimeAsync(999) + expect(sent()).toEqual([1n]) + await vi.advanceTimersByTimeAsync(1) + expect(sent()).toEqual([1n, 2n]) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('sends a count-full batch before the interval expires', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxInflightCount: 2, + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + writer.write(bytes(1)) + await settle() + expect( + stream.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ).toHaveLength(0) + writer.write(bytes(2)) + let request = await stream.waitForWrite() + expect(request.messages.map((message) => message.seqNo)).toEqual([1n, 2n]) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('sends a byte-full batch when the next message cannot fit', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + // Two 25 MiB payloads exceed the 48 MiB batch target. + writer.write(new Uint8Array(25 * 1024 * 1024)) + await settle() + expect( + stream.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ).toHaveLength(0) + writer.write(new Uint8Array(25 * 1024 * 1024)) + let request = await stream.waitForWrite() + expect(request.messages.map((message) => message.seqNo)).toEqual([1n]) + await settle() + expect( + stream.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ).toHaveLength(1) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('sends an overdue partial batch as soon as acknowledgments free the window', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxInflightCount: 2, + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + writer.write(bytes(1)) + writer.write(bytes(2)) + await stream.waitForWrite() + writer.write(bytes(3)) + await vi.advanceTimersByTimeAsync(1000) + let sent = () => + stream.sent.flatMap((frame) => + frame.clientMessage.case === 'writeRequest' + ? frame.clientMessage.value.messages.map((message) => message.seqNo) + : [] + ) + expect(sent()).toEqual([1n, 2n]) + stream.respond(writeResponse([{ seqNo: 1n }, { seqNo: 2n }])) + await settle() + expect(sent()).toEqual([1n, 2n, 3n]) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('flush and close send a partial batch without waiting for its interval', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await settle() + writer.write(bytes(1)) + let flushed = writer.flush() + await stream.waitForWrite() + stream.respond(writeResponse([{ seqNo: 1n }])) + await expect(flushed).resolves.toBe(1n) + writer.write(bytes(2)) + let closed = writer.close() + await settle() + let sent = stream.sent.flatMap((frame) => + frame.clientMessage.case === 'writeRequest' + ? frame.clientMessage.value.messages.map((message) => message.seqNo) + : [] + ) + expect(sent).toEqual([1n, 2n]) + stream.respond(writeResponse([{ seqNo: 2n }])) + await closed + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('retains an overdue batch across reconnect without sending before initialization', async () => { + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxInflightCount: 2, + flushIntervalMs: 1000, + }) + let first = await waitForNextStream() + first.respond(initResponse(0n)) + writer.write(bytes(1)) + writer.write(bytes(2)) + await first.waitForWrite() + writer.write(bytes(3)) + await vi.advanceTimersByTimeAsync(1000) + first.disconnect() + await settle() + await vi.advanceTimersByTimeAsync(50) + let second = await waitForNextStream() + await second.waitForInit() + await vi.advanceTimersByTimeAsync(1000) + expect( + second.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ).toHaveLength(0) + second.respond(initResponse(2n)) + await settle() + let request = await second.waitForWrite() + expect(request.messages.map((message) => message.seqNo)).toEqual([3n]) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('sends a buffered partial batch despite reconnects shorter than the flush interval', async () => { + // Each stream ends before its flush tick, making timer postponement deterministic. + vi.useFakeTimers() + try { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + flushIntervalMs: 1000, + }) + let stream = await waitForNextStream() + let streams = [stream] + stream.respond(initResponse(0n)) + await settle() + writer.write(bytes(1)) + + for (let attempt = 0; attempt < 3; attempt++) { + // oxlint-disable-next-line no-await-in-loop + await vi.advanceTimersByTimeAsync(450) + stream.disconnect() + // oxlint-disable-next-line no-await-in-loop + await settle() + // oxlint-disable-next-line no-await-in-loop + await vi.advanceTimersByTimeAsync(50) + // oxlint-disable-next-line no-await-in-loop + stream = await waitForNextStream() + streams.push(stream) + stream.respond(initResponse(0n)) + // oxlint-disable-next-line no-await-in-loop + await settle() + } + + let sent = streams.flatMap((connection) => + connection.sent.filter((frame) => frame.clientMessage.case === 'writeRequest') + ) + expect(sent.length).toBeGreaterThan(0) + } finally { + vi.clearAllTimers() + vi.useRealTimers() + } +}) + +test('handles a rejected async onAck without delaying flush', async () => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + let observer = Promise.withResolvers() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + onAck: async () => { + await observer.promise + throw new Error('Asynchronous ACK observer failed') + }, + }) + writer.write(bytes(1)) + let flushed = writer.flush() + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + await stream.waitForWrite() + stream.respond(writeResponse([{ seqNo: 1n }])) + await expect(flushed).resolves.toBe(1n) + observer.resolve() + await new Promise((resolve) => setImmediate(resolve)) + await expect(writer.close()).resolves.toBeUndefined() +}) + +// Controlled ACKs separate flush boundaries even when all writes share a stream and batch. +test.for([ + { mode: 'auto', recovery: 'ack' }, + { mode: 'auto', recovery: 'resend' }, + { mode: 'auto', recovery: 'dedup' }, + { mode: 'manual', recovery: 'ack' }, + { mode: 'manual', recovery: 'resend' }, + { mode: 'manual', recovery: 'dedup' }, +])( + 'settles each flush at its captured write boundary ($mode, $recovery)', + async ({ mode, recovery }, tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { + topic: '/t', + producer: 'p', + maxInflightCount: 3, + }) + let seq = mode === 'manual' ? [10n, 20n, 30n] : [1n, 2n, 3n] + writer.write(bytes(1), mode === 'manual' ? { seqNo: seq[0] } : undefined) + let firstDone = false + let first = writer.flush(tc.signal).then((seqNo) => { + firstDone = true + return seqNo + }) + let sameBoundary = writer.flush(tc.signal) + writer.write(bytes(2), mode === 'manual' ? { seqNo: seq[1] } : undefined) + let secondDone = false + let second = writer.flush(tc.signal).then((seqNo) => { + secondDone = true + return seqNo + }) + writer.write(bytes(3), mode === 'manual' ? { seqNo: seq[2] } : undefined) + + let stream = await waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse(0n)) + await stream.waitForWrite() + + if (recovery !== 'ack') { + stream.disconnect() + stream = await waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse(recovery === 'dedup' ? seq[0]! : 0n)) + await stream.waitForWrite() + } + if (recovery !== 'dedup') { + stream.respond(writeResponse([{ seqNo: seq[0]! }])) + } + await settle() + + expect(firstDone).toBe(true) + expect(secondDone).toBe(false) + await expect(first).resolves.toBe(seq[0]) + await expect(sameBoundary).resolves.toBe(seq[0]) + + stream.respond(writeResponse([{ seqNo: seq[1]! }])) + await expect(second).resolves.toBe(seq[1]) + let thirdDone = false + let third = writer.flush(tc.signal).then((seqNo) => { + thirdDone = true + return seqNo + }) + await settle() + expect(thirdDone).toBe(false) + stream.respond(writeResponse([{ seqNo: seq[2]! }])) + await expect(third).resolves.toBe(seq[2]) + await writer.close() + } +) + +test('borrows a whole RAW buffer and permits reuse after acknowledgment', async (tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p' }) + let stream = await waitForNextStream() + await stream.waitForInit() + stream.respond(initResponse(0n)) + + let payload = bytes(1, 2, 3) + writer.write(payload) + let flushed = writer.flush(tc.signal) + let first = await stream.waitForWrite() + expect(first.messages[0]!.data).toBe(payload) + stream.respond(writeResponse([{ seqNo: 1n }])) + await flushed + + payload.fill(4) + writer.write(payload) + let reused = writer.flush(tc.signal) + await settle() + let requests = stream.sent.flatMap((frame) => + frame.clientMessage.case === 'writeRequest' ? frame.clientMessage.value.messages : [] + ) + expect(requests[1]!.seqNo).toBe(2n) + expect([...requests[1]!.data]).toEqual([4, 4, 4]) + + stream.respond(writeResponse([{ seqNo: 2n }])) + await expect(reused).resolves.toBe(2n) +}) + +// Controlled ACKs keep both writes pending across the same reconnect. +test.for([ + { name: 'RAW', codec: RAW_CODEC }, + { name: 'GZIP', codec: GZIP_CODEC }, +])('resends writes sharing one unchanged input buffer with $name', async ({ codec }, tc) => { + let { driver, waitForNextStream } = makeFakeTopicDriver() + using writer = createTopicWriter(driver, { topic: '/t', producer: 'p', codec }) + let first = await waitForNextStream() + await first.waitForInit() + first.respond(initResponse(0n)) + + let payload = bytes(1, 2, 3) + writer.write(payload) + writer.write(payload) + let flushed = writer.flush(tc.signal) + await first.waitForWrite() + first.disconnect() + + let replacement = await waitForNextStream() + await replacement.waitForInit() + replacement.respond(initResponse(0n)) + let resent = await replacement.waitForWrite() + expect(resent.messages.map((message) => message.seqNo)).toEqual([1n, 2n]) + expect(resent.messages.map((message) => [...codec.decompress(message.data)])).toEqual([ + [1, 2, 3], + [1, 2, 3], + ]) + expect([...payload]).toEqual([1, 2, 3]) + expect(resent.messages.every((message) => message.data === payload)).toBe(codec === RAW_CODEC) + + replacement.respond(writeResponse([{ seqNo: 1n }, { seqNo: 2n }])) + await expect(flushed).resolves.toBe(2n) +}) diff --git a/packages/topic/src/writer/writer.memory.test.ts b/packages/topic/src/writer/writer.memory.test.ts new file mode 100644 index 00000000..c800b477 --- /dev/null +++ b/packages/topic/src/writer/writer.memory.test.ts @@ -0,0 +1,89 @@ +import { setTimeout as sleep } from 'node:timers/promises' + +import type { Driver } from '@ydbjs/core' +import { type TestContext, expect, test } from 'vitest' + +import { RAW_CODEC } from '../codec.ts' +import { initResponse, makeFakeTopicDriver, writeResponse } from './writer.fixtures.ts' +import { type TopicWriter, createTopicWriter } from './writer.ts' + +let capturingWriter = function capturingWriter( + driver: Driver, + owner: 'onAck' | 'codec', + registry: FinalizationRegistry, + acks: bigint[] +): TopicWriter { + let captured = new Uint8Array(16 * 1024 * 1024) + registry.register(captured.buffer, undefined) + return createTopicWriter(driver, { + topic: '/t', + onAck: + owner === 'onAck' + ? (seqNo) => { + if (captured[0] !== 0) { + throw new Error('Unexpected callback state') + } + acks.push(seqNo) + } + : (seqNo) => { + acks.push(seqNo) + }, + codec: + owner === 'codec' + ? { + codec: RAW_CODEC.codec, + compress(data) { + if (captured[0] !== 0) { + throw new Error('Unexpected codec state') + } + return data + }, + decompress: RAW_CODEC.decompress, + } + : RAW_CODEC, + }) +} + +// The fake releases the final ACK deterministically; the assertion measures reachability, +// independent of gRPC allocations, RSS caching, or whether the writer itself gets collected. +let verifyReleasedCapture = async function verifyReleasedCapture( + owner: 'onAck' | 'codec', + tc: TestContext +): Promise<{ collected: boolean; acks: bigint[]; writer: TopicWriter }> { + let collected = false + let registry = new FinalizationRegistry(() => { + collected = true + }) + let { driver, waitForNextStream } = makeFakeTopicDriver() + let acks: bigint[] = [] + using writer = capturingWriter(driver, owner, registry, acks) + let stream = await waitForNextStream() + stream.respond(initResponse(0n)) + writer.write(new Uint8Array([1])) + let closed = writer.close(tc.signal) + await stream.waitForWrite() + stream.respond(writeResponse([{ seqNo: 1n }])) + await closed + + // The registry callback updates this condition between GC rounds. + // oxlint-disable-next-line no-unmodified-loop-condition + for (let attempt = 0; attempt < 40 && !collected; attempt++) { + globalThis.gc!() + // oxlint-disable-next-line no-await-in-loop + await sleep(10, undefined, { signal: tc.signal }) + } + await writer.close(tc.signal) + return { collected, acks, writer } +} + +test('releases onAck captures while a closed writer remains reachable', async (tc) => { + let result = await verifyReleasedCapture('onAck', tc) + expect(result).toMatchObject({ collected: true, acks: [1n] }) + await expect(result.writer.close(tc.signal)).resolves.toBeUndefined() +}) + +test('releases custom codec captures while a closed writer remains reachable', async (tc) => { + let result = await verifyReleasedCapture('codec', tc) + expect(result).toMatchObject({ collected: true, acks: [1n] }) + await expect(result.writer.close(tc.signal)).resolves.toBeUndefined() +}) diff --git a/packages/topic/src/writer/writer.model.test.ts b/packages/topic/src/writer/writer.model.test.ts index 21a10237..92bfc822 100644 --- a/packages/topic/src/writer/writer.model.test.ts +++ b/packages/topic/src/writer/writer.model.test.ts @@ -13,24 +13,11 @@ import { createWriterCtx, writerTransition, } from './writer-state.ts' -import { - type TransportEffect, - type TransportEvent, - type TransportOutput, - type TransportState, - transportTransition, -} from './transport-state.ts' import type { WriteAck } from './types.ts' -// Model-based / property test. It wires the two REAL pure transitions -// (writerTransition + transportTransition) to a protocol-faithful server model -// and drives them with random sequences of user calls, network events and timer -// firings, checking a set of invariants after every step. Timers are logical -// (fired on demand), so races the wall clock never surfaces — e.g. start_timeout -// firing just before a slow init — are explored deterministically. -// -// The point (per "test behavior, not code"): instead of hand-picking scenarios, -// we assert the CONTRACT holds across the whole reachable space of orderings. +// The real writer transition runs against an independent server model: messages +// persist by seqNo, acknowledgments arrive in order, and a stream drop loses pending +// replies. Logical timers let a watchdog race an init response deterministically. // Deterministic PRNG so any failure is reproducible from its seed. let mulberry32 = function mulberry32(seed: number): () => number { @@ -46,12 +33,10 @@ let mulberry32 = function mulberry32(seed: number): () => number { let NEVER = new AbortController().signal type Sim = { - // FSMs (real transitions, mutable state + ctx) + // Real writer transition and its event mailbox writerState: WriterState writerCtx: WriterCtx - transportState: TransportState writerEvents: WriterEvent[] - transportEvents: TransportEvent[] // logical timers armed: Set @@ -59,7 +44,6 @@ type Sim = { // transport/network + server model streamOpen: boolean initPending: boolean - pendingGetLastSeqNo: boolean // set by the writer's connect effect streamGetLastSeqNo: boolean // captured when a stream opens pendingAcks: WriteAck[] // received on the current stream, not yet delivered persisted: Set // server dedup store (survives reconnects) @@ -73,7 +57,9 @@ type Sim = { assigned: WeakMap // detect renumbering prevLastSeqNo: bigint modelBytes: bigint // facade-style un-acked byte budget - outstandingFlush: boolean + acknowledgedCount: number + nextFlushId: number + flushBoundaries: Map errored: boolean destroyed: boolean terminal: boolean @@ -87,13 +73,10 @@ let mkSim = function mkSim(maxInflightCount: number, maxBatchBytes: bigint, preS return { writerState: 'idle', writerCtx: createWriterCtx({ maxInflightCount, maxBatchBytes }), - transportState: 'idle', writerEvents: [], - transportEvents: [], armed: new Set(), streamOpen: false, initPending: false, - pendingGetLastSeqNo: false, streamGetLastSeqNo: false, pendingAcks: [], persisted, @@ -105,7 +88,9 @@ let mkSim = function mkSim(maxInflightCount: number, maxBatchBytes: bigint, preS assigned: new WeakMap(), prevLastSeqNo: 0n, modelBytes: 0n, - outstandingFlush: false, + acknowledgedCount: 0, + nextFlushId: 0, + flushBoundaries: new Map(), errored: false, destroyed: false, terminal: false, @@ -113,7 +98,7 @@ let mkSim = function mkSim(maxInflightCount: number, maxBatchBytes: bigint, preS } let allDrained = function allDrained(ctx: WriterCtx): boolean { - return ctx.bufferLength === 0 && ctx.inflightLength === 0 + return ctx.messages.length === 0 } let timerEvent = function timerEvent(which: GlobalTimerName): WriterEvent { @@ -133,30 +118,6 @@ let timerEvent = function timerEvent(which: GlobalTimerName): WriterEvent { } } -// Mirror of writer-runtime.ts mapTransportOutput (kept trivial + local on purpose). -let mapTransportOutput = function mapTransportOutput(output: TransportOutput): WriterEvent | null { - switch (output.type) { - case 'transport.stream.init_response': - return { - type: 'writer.stream.init_response', - sessionId: output.sessionId, - lastSeqNo: output.lastSeqNo, - ...(output.partitionId !== undefined && { partitionId: output.partitionId }), - } - case 'transport.stream.write_response': - return { type: 'writer.stream.write_response', acks: output.acks } - case 'transport.stream.token_response': - return { type: 'writer.stream.token_response' } - case 'transport.stream.disconnected': - return { - type: 'writer.stream.disconnected', - ...('error' in output ? { error: output.error } : {}), - } - default: - return null - } -} - let serverReceive = function serverReceive(sim: Sim, seqNo: bigint): void { let wasNew = !sim.persisted.has(seqNo) if (wasNew) { @@ -175,9 +136,20 @@ let serverReceive = function serverReceive(sim: Sim, seqNo: bigint): void { let onWriterOutput = function onWriterOutput(sim: Sim, output: WriterOutput): void { switch (output.type) { case 'writer.flushed': - sim.outstandingFlush = false + for (let [id, acceptedCount] of sim.flushBoundaries) { + if (id > output.requestId) { + break + } + if (sim.acknowledgedCount < acceptedCount) { + throw new Error( + `flush ${id} completed before its ${acceptedCount} writes were acknowledged` + ) + } + sim.flushBoundaries.delete(id) + } break case 'writer.acknowledgments': + sim.acknowledgedCount += output.acknowledgments.size sim.modelBytes -= output.freedBytes if (sim.modelBytes < 0n) { sim.modelBytes = 0n @@ -196,8 +168,10 @@ let onWriterOutput = function onWriterOutput(sim: Sim, output: WriterOutput): vo let applyWriterEffect = function applyWriterEffect(sim: Sim, effect: WriterEffect): void { switch (effect.type) { case 'writer.effect.transport.connect': - sim.pendingGetLastSeqNo = effect.getLastSeqNo - sim.transportEvents.push({ type: 'transport.connect' }) + sim.streamOpen = true + sim.initPending = true + sim.pendingAcks = [] + sim.streamGetLastSeqNo = effect.getLastSeqNo break case 'writer.effect.send.write_request': // A batch only reaches the server over a live, initialized stream. @@ -209,12 +183,9 @@ let applyWriterEffect = function applyWriterEffect(sim: Sim, effect: WriterEffec break case 'writer.effect.send.update_token': if (sim.streamOpen && !sim.initPending) { - sim.transportEvents.push({ type: 'transport.token' }) + sim.writerEvents.push({ type: 'writer.stream.token_response' }) } break - case 'writer.effect.transport.close': - sim.transportEvents.push({ type: 'transport.close' }) - break case 'writer.effect.timer.schedule': // recovery_window is armed once per reconnect saga (runtime guard). if (effect.which === 'recovery_window' && sim.armed.has('recovery_window')) { @@ -227,26 +198,23 @@ let applyWriterEffect = function applyWriterEffect(sim: Sim, effect: WriterEffec break case 'writer.effect.finalize': sim.armed.clear() - sim.transportEvents.push({ type: 'transport.destroy', reason: effect.reason }) + closeStream(sim) break } } -let applyTransportEffect = function applyTransportEffect(sim: Sim, effect: TransportEffect): void { - switch (effect.type) { - case 'transport.effect.open_stream': - sim.streamOpen = true - sim.initPending = true - sim.pendingAcks = [] - sim.streamGetLastSeqNo = sim.pendingGetLastSeqNo - break - case 'transport.effect.close_stream': - case 'transport.effect.finalize': - sim.streamOpen = false - sim.initPending = false - sim.pendingAcks = [] - break - } +let closeStream = function closeStream(sim: Sim): void { + sim.streamOpen = false + sim.initPending = false + sim.pendingAcks = [] +} + +let disconnect = function disconnect(sim: Sim, error?: unknown): void { + closeStream(sim) + sim.writerEvents.push({ + type: 'writer.stream.disconnected', + ...(error !== undefined && { error }), + }) } let processWriterEvent = function processWriterEvent(sim: Sim, event: WriterEvent): void { @@ -265,43 +233,26 @@ let processWriterEvent = function processWriterEvent(sim: Sim, event: WriterEven } } -let processTransportEvent = function processTransportEvent(sim: Sim, event: TransportEvent): void { - let runtime = { - state: sim.transportState, - signal: NEVER, - emit: (output: TransportOutput) => { - let mapped = mapTransportOutput(output) - if (mapped) { - sim.writerEvents.push(mapped) - } - }, - dispatch: (next: TransportEvent) => sim.transportEvents.push(next), - } - let result = transportTransition({}, event, runtime) - if (result?.state) { - sim.transportState = result.state - } - for (let effect of result?.effects ?? []) { - applyTransportEffect(sim, effect) - } -} - let runToQuiescence = function runToQuiescence(sim: Sim): void { let guard = 0 - while (sim.writerEvents.length > 0 || sim.transportEvents.length > 0) { + while (sim.writerEvents.length > 0) { if (++guard > 200_000) { throw new Error('livelock: quiescence never reached (self-dispatch loop)') } - if (sim.writerEvents.length > 0) { - processWriterEvent(sim, sim.writerEvents.shift()!) - } else { - processTransportEvent(sim, sim.transportEvents.shift()!) - } + processWriterEvent(sim, sim.writerEvents.shift()!) } } let checkInvariants = function checkInvariants(sim: Sim, where: string): void { let ctx = sim.writerCtx + let unsentBytes = ctx.messages + .slice(ctx.inflightCount) + .reduce((sum, message) => sum + BigInt(message.data.length), 0n) + if (ctx.unsentBytes !== unsentBytes) { + throw new Error( + `${where}: unsent payload bytes ${ctx.unsentBytes} differ from queued messages ${unsentBytes}` + ) + } if (sim.terminal) { // Terminal stop releases the buffer and the facade budget resets. @@ -311,22 +262,21 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { if (sim.modelBytes !== 0n) { throw new Error(`${where}: terminal but modelBytes=${sim.modelBytes}`) } + if ( + !sim.errored && + !sim.destroyed && + sim.persisted.size !== sim.preSeeded + sim.writtenCount + ) { + throw new Error( + `${where}: graceful close lost writes: persisted=${sim.persisted.size}, expected=${sim.preSeeded + sim.writtenCount}` + ) + } return } let n = ctx.messages.length - if (!(ctx.inflightStart >= 0 && ctx.inflightStart <= ctx.bufferStart && ctx.bufferStart <= n)) { - throw new Error( - `${where}: pointer invariant broken inflightStart=${ctx.inflightStart} bufferStart=${ctx.bufferStart} len=${n}` - ) - } - if (ctx.bufferLength !== n - ctx.bufferStart) { - throw new Error(`${where}: bufferLength=${ctx.bufferLength} != ${n - ctx.bufferStart}`) - } - if (ctx.inflightLength !== ctx.bufferStart - ctx.inflightStart) { - throw new Error( - `${where}: inflightLength=${ctx.inflightLength} != ${ctx.bufferStart - ctx.inflightStart}` - ) + if (ctx.inflightCount < 0 || ctx.inflightCount > n) { + throw new Error(`${where}: invalid in-flight prefix ${ctx.inflightCount} for ${n} messages`) } if (ctx.lastSeqNo < sim.prevLastSeqNo) { @@ -355,25 +305,22 @@ let checkInvariants = function checkInvariants(sim: Sim, where: string): void { } } - // The byte budget and flush-liveness only apply while the facade budget is - // authoritative — i.e. in the live, write-accepting states. The closing drain - // keeps emitting acks and resolves a pending flush once drained, but the model's - // step generator does not track the drain-vs-terminate race precisely enough to - // assert liveness there; the facade also resets #bufferedBytes to 0 on close. - if (sim.writerState !== 'closing') { - let expected = 0n - for (let i = ctx.inflightStart; i < n; i++) { - expected += BigInt(ctx.messages[i]!.data.length) - } - if (sim.modelBytes !== expected) { + let expected = 0n + for (let i = 0; i < n; i++) { + expected += BigInt(ctx.messages[i]!.data.length) + } + if (sim.modelBytes !== expected) { + throw new Error( + `${where}: byte budget drift modelBytes=${sim.modelBytes} expected=${expected}` + ) + } + + for (let [id, acceptedCount] of sim.flushBoundaries) { + if (sim.acknowledgedCount >= acceptedCount) { throw new Error( - `${where}: byte budget drift modelBytes=${sim.modelBytes} expected=${expected}` + `${where}: flush ${id} unresolved after its ${acceptedCount} writes were acknowledged` ) } - - if (sim.outstandingFlush && allDrained(ctx)) { - throw new Error(`${where}: flush left unresolved though the window is drained`) - } } } @@ -401,22 +348,23 @@ let cooldown = function cooldown(sim: Sim): void { sim.initPending = false sim.sessions++ // Healthy recovery in cooldown: always report the true high-water mark. - sim.transportEvents.push({ - type: 'transport.init', + sim.writerEvents.push({ + type: 'writer.stream.init_response', sessionId: `s${sim.sessions}`, lastSeqNo: sim.serverLastSeqNo, }) runToQuiescence(sim) continue } - if (sim.writerCtx.bufferLength > 0 && sim.armed.has('flush_tick')) { - sim.writerEvents.push(timerEvent('flush_tick')) // triggers pump + + if (sim.armed.has('flush_tick')) { + sim.writerEvents.push(timerEvent('flush_tick')) } runToQuiescence(sim) if (sim.pendingAcks.length > 0) { let acks = sim.pendingAcks sim.pendingAcks = [] - sim.transportEvents.push({ type: 'transport.write', acks }) + sim.writerEvents.push({ type: 'writer.stream.write_response', acks }) runToQuiescence(sim) } } @@ -444,10 +392,11 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { let lastAction = 'start' for (let step = 0; step < cfg.steps && !sim.terminal; step++) { let live = - sim.writerState === 'idle' || - sim.writerState === 'connecting' || - sim.writerState === 'ready' || - sim.writerState === 'reconnecting' + !sim.writerCtx.closeRequested && + (sim.writerState === 'idle' || + sim.writerState === 'connecting' || + sim.writerState === 'ready' || + sim.writerState === 'reconnecting') let actions: Array<{ w: number; name: string; run: () => void }> = [] @@ -456,7 +405,7 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { w: 10, name: 'write', run: () => { - let size = 1 + randInt(6) + let size = 1 + randInt(Math.min(6, Number(cfg.maxBatchBytes))) let data = new Uint8Array(size) let seqNo = 0n if (cfg.mode === 'manual') { @@ -478,8 +427,9 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { w: 3, name: 'flush', run: () => { - sim.outstandingFlush = true - sim.writerEvents.push({ type: 'writer.flush' }) + let requestId = ++sim.nextFlushId + sim.flushBoundaries.set(requestId, sim.writtenCount) + sim.writerEvents.push({ type: 'writer.flush', requestId }) }, }) actions.push({ @@ -512,8 +462,8 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { // and one that reports 0 (messages are resent). let reported = sim.streamGetLastSeqNo || randInt(2) === 0 ? sim.serverLastSeqNo : 0n - sim.transportEvents.push({ - type: 'transport.init', + sim.writerEvents.push({ + type: 'writer.stream.init_response', sessionId: `s${sim.sessions}`, lastSeqNo: reported, }) @@ -528,7 +478,7 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { // The server acks an in-order prefix of what it received. let k = 1 + randInt(sim.pendingAcks.length) let acks = sim.pendingAcks.splice(0, k) - sim.transportEvents.push({ type: 'transport.write', acks }) + sim.writerEvents.push({ type: 'writer.stream.write_response', acks }) }, }) } @@ -536,25 +486,17 @@ let runOne = function runOne(seed: number, cfg: RunConfig): void { actions.push({ w: 3, name: 'drop', - run: () => sim.transportEvents.push({ type: 'transport.ended' }), + run: () => disconnect(sim), }) actions.push({ w: 3, name: 'fail-retryable', - run: () => - sim.transportEvents.push({ - type: 'transport.error', - error: new YDBError(StatusIds_StatusCode.UNAVAILABLE, []), - }), + run: () => disconnect(sim, new YDBError(StatusIds_StatusCode.UNAVAILABLE, [])), }) actions.push({ w: 1, name: 'fail-fatal', - run: () => - sim.transportEvents.push({ - type: 'transport.error', - error: new YDBError(StatusIds_StatusCode.SCHEME_ERROR, []), - }), + run: () => disconnect(sim, new YDBError(StatusIds_StatusCode.SCHEME_ERROR, [])), }) } for (let which of sim.armed) { diff --git a/packages/topic/src/writer/writer.semantics.test.ts b/packages/topic/src/writer/writer.semantics.test.ts index ed3bdeeb..c3af938b 100644 --- a/packages/topic/src/writer/writer.semantics.test.ts +++ b/packages/topic/src/writer/writer.semantics.test.ts @@ -4,9 +4,9 @@ import { YDBError } from '@ydbjs/error' import { ClientError, Status } from 'nice-grpc' import { expect, test } from 'vitest' +import { isRetryableTopicError } from '../retry.ts' import { GZIP_CODEC } from '../codec.ts' import type { TX } from '../tx.ts' -import { isRetryableWriterError } from './writer-state.ts' import { failureResponse, initResponse, @@ -141,13 +141,11 @@ let classifierTable: Array<{ error: new ClientError(GRPC_PATH, Status.CANCELLED, ''), retryable: true, }, - // Divergence from the go SDK: go treats DEADLINE_EXCEEDED on a topic stream as - // retryable (a proxy/LB deadline blip reconnects), while this classifier is - // terminal — the writer is destroyed. Pinned as currently implemented. + { - name: 'classifies a DEADLINE_EXCEEDED transport error as fatal', + name: 'retries a server deadline on an idempotent topic stream', error: new ClientError(GRPC_PATH, Status.DEADLINE_EXCEEDED, ''), - retryable: false, + retryable: true, }, { name: 'classifies an UNAVAILABLE transport error as retryable', @@ -157,7 +155,7 @@ let classifierTable: Array<{ ] test.each(classifierTable)('$name', ({ error, retryOnSchemeError, retryable }) => { - expect(isRetryableWriterError(error, retryOnSchemeError ?? false)).toBe(retryable) + expect(isRetryableTopicError(error, retryOnSchemeError ?? false)).toBe(retryable) }) // ── public constructor producer id ─────────────────────────────────────────────── diff --git a/packages/topic/src/writer/writer.ts b/packages/topic/src/writer/writer.ts index 44f49599..209bac86 100644 --- a/packages/topic/src/writer/writer.ts +++ b/packages/topic/src/writer/writer.ts @@ -1,10 +1,12 @@ +import { randomUUID } from 'node:crypto' +import { isPromise } from 'node:util/types' + import { abortable } from '@ydbjs/abortable' import type { Driver } from '@ydbjs/core' import { loggers } from '@ydbjs/debug' import { type CompressionCodec, RAW_CODEC } from '../codec.js' import type { TX } from '../tx.js' -import { generateProducerId } from './producer-id.js' import { type WriterScope, ackBreakdown, @@ -34,6 +36,8 @@ import type { OnAckCallback, TopicWriterOptions, WriteExtra } from './types.js' // per-message noise (that lives under `ydb:topic:writer:event`). let dbg = loggers.topic.extend('writer') +let MAX_SEQ_NO = (1n << 63n) - 1n + // Synchronous seqNo validator, owned by the facade so write() can reject bad // input at the call site without racing the FSM's async event queue. class SeqNoValidator { @@ -43,6 +47,9 @@ class SeqNoValidator { // Returns the message seqNo (0n means "assign at send time" in auto mode). validate(userSeqNo: bigint | undefined): bigint { let provided = userSeqNo !== undefined + if (provided && (userSeqNo! < 1n || userSeqNo! > MAX_SEQ_NO)) { + throw new RangeError('seqNo must be a positive int64') + } if (this.#mode === null) { this.#mode = provided ? 'manual' : 'auto' @@ -81,7 +88,9 @@ export class TopicWriter implements AsyncDisposable, Disposable { #codec: CompressionCodec #runtime: WriterRuntime #validator = new SeqNoValidator() - #flushWaiters: Array> = [] + #flushWaiters = new Map>() + // Per-writer flush call order, independent of message seqNos and gRPC requests. + #nextFlushId = 0 // Byte budget mirrored here so write() can reject a full buffer synchronously, // ahead of the FSM's async mailbox. Incremented on write, decremented as the @@ -123,7 +132,7 @@ export class TopicWriter implements AsyncDisposable, Disposable { if (options.producer === '') { throw new Error('producer must be a non-empty string — omit it to get a generated id') } - options = { ...options, producer: options.producer ?? generateProducerId() } + options = { ...options, producer: options.producer ?? randomUUID() } this.#onAck = options.onAck this.#transactional = options.tx !== undefined @@ -131,8 +140,7 @@ export class TopicWriter implements AsyncDisposable, Disposable { this.#maxBufferBytes = options.maxBufferBytes ?? DEFAULT_MAX_BUFFER_BYTES this.#scope = { driver: driver.identity, topic: options.topic, producer: options.producer! } - // One-shot effective-config snapshot for late-joining metrics/traces - // subscribers — built from the runtime's own defaults, correct by construction. + // Publish the configuration once when the writer is created. publishOpened(this.#scope, { codec: this.#codec.codec, maxInflightCount: options.maxInflightCount ?? DEFAULT_MAX_INFLIGHT_COUNT, @@ -172,11 +180,12 @@ export class TopicWriter implements AsyncDisposable, Disposable { } } + /** RAW buffers may be shared by writes; keep their bytes unchanged until all corresponding ACKs. */ write(data: Uint8Array, extra?: WriteExtra): void { if (this.#closed || this.#closing) { throw new Error('Writer is closed — cannot write messages') } - if (this.#lastError) { + if (this.#lastError !== undefined) { throw new Error('Writer has failed — cannot write messages', { cause: this.#lastError }) } // Size limit applies to the uncompressed payload. @@ -186,18 +195,40 @@ export class TopicWriter implements AsyncDisposable, Disposable { ) } + let createdAt = new Date(extra?.createdAt?.getTime() ?? Date.now()) + if (!Number.isFinite(createdAt.getTime())) { + throw new RangeError('createdAt must be a valid date') + } + let uncompressedSize = BigInt(data.length) let payload = this.#codec.compress(data) let bufferedSize = BigInt(payload.length) - // Fail-fast cap on retained (un-acknowledged) bytes to bound memory. Checked - // before the seqNo validator mutates, so a rejected write leaves no state behind. + // Check the compressed-payload budget before consuming a sequence number. if (this.#bufferedBytes + bufferedSize > this.#maxBufferBytes) { throw new Error( `Writer buffer is full: ${this.#bufferedBytes + bufferedSize} bytes would exceed the ${this.#maxBufferBytes} byte limit` ) } + // Copy short or growable views so accepted data cannot retain an unbudgeted backing allocation. + let backing = payload.buffer + if ( + payload.byteLength !== backing.byteLength || + ('resizable' in backing && backing.resizable) || + ('growable' in backing && backing.growable) + ) { + payload = new Uint8Array(payload) + } + let metadataItems = extra?.metadataItems + ? Object.fromEntries( + Object.entries(extra.metadataItems).map(([key, value]) => [ + key, + new Uint8Array(value), + ]) + ) + : undefined + let seqNo = this.#validator.validate(extra?.seqNo) this.#bufferedBytes += bufferedSize @@ -207,14 +238,15 @@ export class TopicWriter implements AsyncDisposable, Disposable { data: payload, uncompressedSize, seqNo, - createdAt: extra?.createdAt ?? new Date(), - ...(extra?.metadataItems && { metadataItems: extra.metadataItems }), + createdAt, + ...(metadataItems && { metadataItems }), }, }) } + /** Waits for ACKs of writes accepted before this call; later writes do not extend the wait. */ async flush(signal?: AbortSignal): Promise { - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } if (this.#closed) { @@ -224,8 +256,9 @@ export class TopicWriter implements AsyncDisposable, Disposable { // One span per flush covers batching + server acks + any reconnect in between. return traceFlush(this.#scope, async () => { let waiter = Promise.withResolvers() - this.#flushWaiters.push(waiter) - this.#runtime.machine.dispatch({ type: 'writer.flush' }) + let requestId = ++this.#nextFlushId + this.#flushWaiters.set(requestId, waiter) + this.#runtime.machine.dispatch({ type: 'writer.flush', requestId }) if (!signal) { return waiter.promise @@ -238,11 +271,8 @@ export class TopicWriter implements AsyncDisposable, Disposable { // waiter so it neither lingers in #flushWaiters — which would grow // unbounded when a long-lived signal is threaded into many flush() calls // — nor later rejects unhandled when the writer terminates. If the FSM - // already removed it (error/closed), indexOf is -1 and this is a no-op. - let index = this.#flushWaiters.indexOf(waiter) - if (index !== -1) { - this.#flushWaiters.splice(index, 1) - } + // already removed it (error/closed), deletion is a no-op. + this.#flushWaiters.delete(requestId) throw error } }) @@ -251,7 +281,7 @@ export class TopicWriter implements AsyncDisposable, Disposable { async close(signal?: AbortSignal): Promise { if (this.#closed) { // A close that dropped data surfaces the failure even on a repeat call. - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } return @@ -265,7 +295,7 @@ export class TopicWriter implements AsyncDisposable, Disposable { await (signal ? abortable(signal, closed) : closed) // The graceful drain failed (errored / timed out with undelivered messages). - if (this.#lastError) { + if (this.#lastError !== undefined) { throw this.#lastError } } @@ -278,9 +308,10 @@ export class TopicWriter implements AsyncDisposable, Disposable { this.#closing = true let error = reason ?? new Error('Writer destroyed') this.#lastError = error - for (let waiter of this.#flushWaiters.splice(0)) { + for (let waiter of this.#flushWaiters.values()) { waiter.reject(error) } + this.#flushWaiters.clear() this.#runtime.machine.dispatch({ type: 'writer.destroy', reason: error }) } @@ -333,7 +364,10 @@ export class TopicWriter implements AsyncDisposable, Disposable { if (this.#onAck) { for (let [seqNo, status] of output.acknowledgments) { try { - this.#onAck(seqNo, status) + let result: unknown = this.#onAck(seqNo, status) + if (isPromise(result)) { + void result.catch((error) => dbg.log('onAck threw: %O', error)) + } } catch (error) { // User callback errors are logged and ignored — a throwing // callback must never break the writer. @@ -344,7 +378,12 @@ export class TopicWriter implements AsyncDisposable, Disposable { break case 'writer.flushed': - for (let waiter of this.#flushWaiters.splice(0)) { + // Later calls can register while this completion waits in the output queue. + for (let [requestId, waiter] of this.#flushWaiters) { + if (requestId > output.requestId) { + break + } + this.#flushWaiters.delete(requestId) waiter.resolve(output.lastSeqNo) } break @@ -358,9 +397,10 @@ export class TopicWriter implements AsyncDisposable, Disposable { dbg.log('errored: %O', output.error) this.#lastError = output.error publishErrored(this.#scope, output.error) - for (let waiter of this.#flushWaiters.splice(0)) { + for (let waiter of this.#flushWaiters.values()) { waiter.reject(output.error) } + this.#flushWaiters.clear() break case 'writer.closed': @@ -387,11 +427,14 @@ export class TopicWriter implements AsyncDisposable, Disposable { return } this.#closed = true + this.#onAck = undefined + this.#codec = RAW_CODEC this.#bufferedBytes = 0n publishClosed(this.#scope) - for (let waiter of this.#flushWaiters.splice(0)) { + for (let waiter of this.#flushWaiters.values()) { waiter.reject(this.#lastError ?? new Error('Writer closed before flush completed')) } + this.#flushWaiters.clear() // Resolved AFTER writer.error was processed (emitted first), so close() // sees #lastError and can reject on an unclean close. this.#closedDeferred.resolve() diff --git a/packages/topic/tests/memory-leak.test.ts b/packages/topic/tests/memory-leak.test.ts index 565de9bd..b3d0851b 100644 --- a/packages/topic/tests/memory-leak.test.ts +++ b/packages/topic/tests/memory-leak.test.ts @@ -1,103 +1,91 @@ -import { afterEach, beforeEach, expect, inject, test } from 'vitest' -import * as v8 from 'node:v8' - -import { create } from '@bufbuild/protobuf' -import { - CreateTopicRequestSchema, - DropTopicRequestSchema, - TopicServiceDefinition, -} from '@ydbjs/api/topic' -import { Driver } from '@ydbjs/core' -import { createTopicReader } from '../src/reader/index.js' - -let driver = new Driver(inject('connectionString'), { - 'ydb.sdk.enable_discovery': false, -}) -await driver.ready() - -let topicService = driver.createClient(TopicServiceDefinition) - -let testTopicName: string -let testConsumerName: string - -beforeEach(async () => { - testTopicName = `test-topic-memory-${Date.now()}` - testConsumerName = `test-consumer-${Date.now()}` - - await topicService.createTopic( - create(CreateTopicRequestSchema, { - path: testTopicName, - partitioningSettings: { - minActivePartitions: 1n, - maxActivePartitions: 100n, +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import { expect, inject, test } from 'vitest' +import { runMemoryProfile, verifyMemoryProfile } from './memory-profile.ts' +import type { MemoryReport } from './memory-worker.ts' + +let execute = promisify(execFile) + +test( + 'releases topic memory after drain, reconnect and client replacement', + { timeout: Number(process.env['YDB_MEMORY_TIMEOUT_MS'] ?? 180_000) }, + async (tc) => { + await using files = { + path: await mkdtemp(join(tmpdir(), 'topic-memory-')), + [Symbol.asyncDispose]() { + return rm(this.path, { recursive: true, force: true }) }, - consumers: [ - { - name: testConsumerName, - }, - ], - }) - ) -}) - -afterEach(async () => { - await topicService.dropTopic( - create(DropTopicRequestSchema, { - path: testTopicName, - }) - ) -}) - -// oxlint-disable-next-line -test.skip('memory leak check', { timeout: 300_000 }, async () => { - let initialMemory = process.memoryUsage().heapUsed - let iterations = 50_000 - - let snapshotPath = v8.writeHeapSnapshot() - console.log(`Heap snapshot written to: ${snapshotPath}`) - - for (let i = 0; i < iterations; i++) { - // oxlint-disable-next-line no-await-in-loop - await using reader = createTopicReader(driver, { - topic: testTopicName, - consumer: testConsumerName, - }) - - try { - // oxlint-disable-next-line no-await-in-loop - for await (let messages of reader.read({ - signal: AbortSignal.timeout(1), // Very short timeout to trigger more errors - })) { - await reader.commit(messages) - } - } catch { - // Ignore timeout errors } - - if (i % 5_000 === 0) { - if (global.gc) { - global.gc() - } - - let finalMemory = process.memoryUsage().heapUsed - let diff = finalMemory - initialMemory - - console.log(`Memory diff (i=${i}): ${diff / 1024 / 1024} MB`) + let bundle = join(files.path, 'memory-worker.mjs') + await execute( + 'bun', + [ + 'build', + fileURLToPath(new URL('./memory-worker.ts', import.meta.url)), + '--target=node', + '--conditions=development', + `--outfile=${bundle}`, + ], + { signal: tc.signal } + ) + let { report, cuts } = await runMemoryProfile( + inject('connectionString'), + bundle, + join(files.path, 'report.json'), + tc.signal + ) + if (process.env['YDB_MEMORY_REPORT_FILE']) { + await writeFile( + process.env['YDB_MEMORY_REPORT_FILE'], + JSON.stringify(report, null, 2) + '\n' + ) } + let expected = 3 * report.epochs * report.messagesPerEpoch + expect(report.accepted).toEqual([expected, expected, expected]) + verifyMemoryProfile(report, cuts) } - - if (global.gc) { - global.gc() +) + +test('rejects Bun buffer retention even when arrayBuffers reports zero', () => { + let report: MemoryReport = { + runtime: { name: 'bun', version: 'fixture', supportsActiveResources: false }, + node: 'compatibility-version', + platform: 'fixture', + arch: 'fixture', + epochs: 9, + epochPauseMs: 0, + payloadBytes: 32 * 1024, + messagesPerEpoch: 128, + accepted: [1, 1, 1], + acknowledged: [1, 1, 1], + delivered: [1, 1, 1], + committed: [1, 1, 1], + reconnects: { reader: 1, writer: 1 }, + closedClientsAlive: 0, + retainedControlBytes: 0, + samples: ['steady', 'reconnect', 'replace'].flatMap((scenario) => + Array.from({ length: 9 }, (_, epoch) => ({ + phase: 'drained', + scenario, + epoch, + rss: 0, + heapUsed: 1024, + heapTotal: 1024, + external: 0, + arrayBuffers: 0, + activeResources: [], + jsc: { heapSize: 1024, extraMemorySize: 1024, objectCount: 100 }, + })) + ), } - - let finalMemory = process.memoryUsage().heapUsed - let diff = finalMemory - initialMemory - - console.log(`Memory diff: ${diff / 1024 / 1024} MB`) - snapshotPath = v8.writeHeapSnapshot() - console.log(`Heap snapshot written to: ${snapshotPath}`) - - // Allow some fluctuation, but it shouldn't be massive - // 10MB is a generous buffer for 100 iterations if there's no leak - expect(diff).toBeLessThan(10 * 1024 * 1024) + expect(() => verifyMemoryProfile(report, 1)).not.toThrow() + for (let sample of report.samples) { + if (sample.epoch >= 6) sample.jsc!.extraMemorySize += 16 * 1024 * 1024 + } + expect(() => verifyMemoryProfile(report, 1)).toThrow(/jsc.extraMemorySize grew/) }) diff --git a/packages/topic/tests/memory-profile.ts b/packages/topic/tests/memory-profile.ts new file mode 100644 index 00000000..14dbcd00 --- /dev/null +++ b/packages/topic/tests/memory-profile.ts @@ -0,0 +1,180 @@ +import assert from 'node:assert/strict' +import { fork } from 'node:child_process' +import { readFile } from 'node:fs/promises' +import { createServer } from 'node:http' +import { type Socket, createConnection } from 'node:net' + +import type { MemoryReport, MemorySample } from './memory-worker.ts' + +export async function runMemoryProfile( + connectionString: string, + workerPath: string, + reportPath: string, + signal?: AbortSignal +): Promise<{ report: MemoryReport; cuts: number }> { + let upstream = new URL(connectionString) + let upstreamPort = Number(upstream.port || (upstream.protocol === 'grpcs:' ? 443 : 80)) + let sockets = new Set() + let track = (socket: Socket) => { + socket.setNoDelay(true) + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + } + let disconnect = () => { + for (let socket of sockets) socket.destroy() + } + let server = createServer() + server.on('connection', track) + server.on('connect', (request, incoming, head) => { + if (request.url !== `${upstream.hostname}:${upstreamPort}`) { + incoming.end('HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n') + return + } + let outgoing = createConnection({ + host: upstream.hostname, + port: upstreamPort, + }) + track(outgoing) + incoming.once('error', () => outgoing.destroy()) + outgoing.once('error', () => incoming.destroy()) + incoming.once('close', () => outgoing.destroy()) + outgoing.once('close', () => incoming.destroy()) + outgoing.once('connect', () => { + incoming.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length) outgoing.write(head) + incoming.pipe(outgoing).pipe(incoming) + }) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + await using proxy = { + server, + async [Symbol.asyncDispose]() { + disconnect() + await new Promise((resolve) => server.close(() => resolve())) + }, + } + let address = proxy.server.address() + if (!address || typeof address === 'string') throw new Error('Missing proxy address') + let reportFile = reportPath + let child = fork(workerPath, [], { + execPath: process.env['YDB_MEMORY_RUNTIME'] || process.execPath, + execArgv: ['--expose-gc', '--max-old-space-size=256'], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + signal, + env: { + ...process.env, + DEBUG: '', + YDB_CONNECTION_STRING: connectionString, + // CONNECT preserves the server hostname for TLS identity verification. + grpc_proxy: `http://127.0.0.1:${address.port}`, + no_grpc_proxy: '', + no_proxy: '', + MEMORY_REPORT_FILE: reportFile, + }, + }) + let output = '' + for (let stream of [child.stdout, child.stderr]) { + stream?.on('data', (chunk) => { + output = (output + String(chunk)).slice(-16_384) + if (process.env['YDB_MEMORY_PROGRESS'] === '1') process.stdout.write(chunk) + }) + } + let cuts = 0 + child.on('message', (message: unknown) => { + if ( + message && + typeof message === 'object' && + 'type' in message && + message.type === 'disconnect' + ) { + cuts++ + disconnect() + child.send({ type: 'disconnected' }) + } + }) + let code = await new Promise((resolve, reject) => { + child.once('error', (error) => { + // Keep the proxy alive until the interrupted worker finishes cleanup. + if (error.name !== 'AbortError') reject(error) + }) + child.once('exit', resolve) + }) + assert.equal(code, 0, output) + let report = JSON.parse(await readFile(reportFile, 'utf8')) as MemoryReport + return { report, cuts } +} + +export function verifyMemoryProfile(report: MemoryReport, cuts: number): void { + let runtime = report.runtime ?? { + name: 'node', + version: report.node, + supportsActiveResources: true, + } + assert(runtime.name === 'node' || runtime.name === 'bun', 'Unknown memory runtime') + assert(report.reconnects.reader >= cuts) + assert(report.reconnects.writer >= cuts) + assert(cuts > 0) + assert.deepEqual(report.accepted, report.delivered) + assert.deepEqual(report.accepted, report.acknowledged) + assert.deepEqual(report.accepted, report.committed) + assert.equal( + report.closedClientsAlive, + 0, + `Closed clients remain reachable: ${JSON.stringify(report.closedClientsRemaining ?? [])}` + ) + + // Compare equally drained post-GC checkpoints; the verifier keeps counters, not message history. + for (let scenario of ['steady', 'reconnect', 'replace']) { + let measured = report.samples.filter( + (sample) => + sample.phase === 'drained' && sample.scenario === scenario && sample.epoch >= 3 + ) + assert(measured.length >= 6) + let head = measured.slice(0, 3) + let tail = measured.slice(-3) + let heap = (sample: MemorySample): number => + runtime.name === 'bun' ? sample.jsc!.heapSize : sample.heapUsed + let buffers = (sample: MemorySample): number => + runtime.name === 'bun' ? sample.jsc!.extraMemorySize : sample.arrayBuffers + let heapMetric = runtime.name === 'bun' ? 'jsc.heapSize' : 'heapUsed' + let bufferMetric = runtime.name === 'bun' ? 'jsc.extraMemorySize' : 'arrayBuffers' + if (runtime.name === 'bun') { + for (let sample of measured) { + assert(sample.jsc, 'Bun memory samples require bun:jsc heap statistics') + assert(Number.isFinite(sample.jsc.heapSize) && sample.jsc.heapSize >= 0) + assert( + Number.isFinite(sample.jsc.extraMemorySize) && sample.jsc.extraMemorySize >= 0 + ) + assert(Number.isSafeInteger(sample.jsc.objectCount) && sample.jsc.objectCount >= 0) + } + } + let mean = (samples: MemorySample[], metric: (sample: MemorySample) => number) => + samples.reduce((sum, sample) => sum + metric(sample), 0) / samples.length + + // RSS includes allocator reserves that GC need not return to the OS. + assert( + mean(tail, (sample) => sample.rss) - mean(head, (sample) => sample.rss) <= + 64 * 1024 * 1024, + `${scenario} rss grew beyond 64 MiB: ${JSON.stringify(measured)}` + ) + + assert( + mean(tail, heap) - mean(head, heap) < 8 * 1024 * 1024, + `${scenario} ${heapMetric} grew beyond 8 MiB: ${JSON.stringify(measured)}` + ) + assert( + mean(tail, buffers) - mean(head, buffers) < 4 * 1024 * 1024, + `${scenario} ${bufferMetric} grew beyond 4 MiB: ${JSON.stringify(measured)}` + ) + } + // Bun's resource inventory is empty even with an open TCP socket. + if (runtime.name === 'node') { + assert.deepEqual( + report.samples.at(-1)!.activeResources.filter((name) => /TCP|TLS/.test(name)), + [] + ) + } +} diff --git a/packages/topic/tests/memory-worker.ts b/packages/topic/tests/memory-worker.ts new file mode 100644 index 00000000..c4ca0da1 --- /dev/null +++ b/packages/topic/tests/memory-worker.ts @@ -0,0 +1,364 @@ +import assert from 'node:assert/strict' +import { randomUUID } from 'node:crypto' +import { channel } from 'node:diagnostics_channel' +import { once } from 'node:events' +import { writeFile } from 'node:fs/promises' +import { setTimeout as sleep, setImmediate as turn } from 'node:timers/promises' + +import { create } from '@bufbuild/protobuf' +import { StatusIds_StatusCode } from '@ydbjs/api/operation' +import { + Codec, + CreateTopicRequestSchema, + DropTopicRequestSchema, + TopicServiceDefinition, +} from '@ydbjs/api/topic' +import { Driver } from '@ydbjs/core' +import { GZIP_CODEC, RAW_CODEC, ZSTD_CODEC } from '../src/codec.ts' +import { type TopicReader, createTopicReader } from '../src/reader/index.ts' +import { type TopicWriter, createTopicWriter } from '../src/writer/index.ts' +import type { TopicMessage } from '../src/message.ts' + +type JscMemory = { heapSize: number; extraMemorySize: number; objectCount: number } +export type MemoryRuntime = { + name: 'node' | 'bun' + version: string + supportsActiveResources: boolean +} +export type MemorySample = ReturnType & { + phase: string + scenario: string + epoch: number + activeResources: string[] + jsc?: JscMemory +} +type ClosedClientLabel = { + role: 'reader' | 'writer' + generation: number + partition?: number + scenario: string + epoch: number +} +export type MemoryReport = { + topic?: string + runtime?: MemoryRuntime + node: string + platform: string + arch: string + epochs: number + epochPauseMs: number + payloadBytes: number + messagesPerEpoch: number + accepted: number[] + acknowledged: number[] + delivered: number[] + committed: number[] + reconnects: { reader: number; writer: number } + closedClientsAlive: number + closedClientsRemaining?: ClosedClientLabel[] + retainedControlBytes: number + samples: MemorySample[] +} +type Clients = { + reader: TopicReader + writers: TopicWriter[] + close(scenario: string, epoch: number): Promise + destroy(): void +} + +let payloadBytes = 32 * 1024 +let messagesPerEpoch = 128 +let epochs = Number(process.env['YDB_MEMORY_EPOCHS'] ?? 15) +assert(Number.isSafeInteger(epochs) && epochs >= 9 && epochs <= 1000) +let epochPauseMs = Number(process.env['YDB_MEMORY_EPOCH_PAUSE_MS'] ?? 0) +assert(Number.isSafeInteger(epochPauseMs) && epochPauseMs >= 0 && epochPauseMs <= 5000) +let bun = (globalThis as { Bun?: { version: string; gc(full: boolean): unknown } }).Bun +let runtime: MemoryRuntime = bun + ? { name: 'bun', version: bun.version, supportsActiveResources: false } + : { name: 'node', version: process.version, supportsActiveResources: true } +let jscHeapStats: (() => JscMemory) | undefined +if (bun) { + // Keep the Bun-only module out of the Node bundle's static dependency graph. + let moduleName = 'bun:jsc' + jscHeapStats = ((await import(moduleName)) as { heapStats: () => JscMemory }).heapStats +} else { + assert.equal(typeof globalThis.gc, 'function', 'Memory worker requires --expose-gc') +} +let collect = () => (bun ? bun.gc(true) : globalThis.gc!()) +let memorySample = (): Omit => { + let usage = process.memoryUsage() + let jsc = jscHeapStats?.() + return { + ...usage, + activeResources: process.getActiveResourcesInfo(), + ...(jsc && { + jsc: { + heapSize: jsc.heapSize, + extraMemorySize: jsc.extraMemorySize, + objectCount: jsc.objectCount, + }, + }), + } +} +// SDK reconnect timers are unref'ed; the harness owns this process until the run finishes. +process.channel?.ref() +let signal = new AbortController() +process.once('SIGTERM', () => signal.abort(new Error('Memory workload interrupted'))) +let codecs = [RAW_CODEC, GZIP_CODEC, ZSTD_CODEC] +let topic = `memory-${randomUUID()}` +console.log(JSON.stringify({ type: 'topic', topic })) +let consumer = 'memory-consumer' +let producers = codecs.map((_, partition) => `${topic}-${partition}`) +let accepted = codecs.map(() => 0) +let acknowledged = codecs.map(() => 0) +let delivered = codecs.map(() => 0) +let committed = codecs.map(() => 0) +let reconnects = { reader: 0, writer: 0 } +let closedClients: Array }> = [] +let generation = 0 +let retained: Uint8Array[] = [] +let samples: MemorySample[] = [] + +let checkpoint = async (phase: string, epoch: number, scenario = 'closed') => { + for (let pass = 0; pass < 3; pass++) { + collect() + // Buffer finalizers and socket shutdown callbacks run between collections. + // oxlint-disable-next-line no-await-in-loop + await turn() + } + let sample = { phase, scenario, epoch, ...memorySample() } + samples.push(sample) + console.log(JSON.stringify({ runtime, ...sample })) +} + +let createPayload = (partition: number, sequence: number): Uint8Array => { + let data = new Uint8Array(payloadBytes).fill((partition + sequence) % 251) + let view = new DataView(data.buffer) + view.setUint32(0, partition) + view.setBigUint64(8, BigInt(sequence)) + return data +} + +// End the read iterator frame before measuring drained or closed client reachability. +async function consumeEpoch(reader: TopicReader): Promise { + let batches: AsyncIterable = reader.read({ + limit: 16, + signal: signal.signal, + }) + // oxlint-disable-next-line no-await-in-loop + for await (let batch of batches) { + for (let message of batch) { + let data = message.payload + assert.equal(data.byteLength, payloadBytes) + let view = new DataView(data.buffer, data.byteOffset, data.byteLength) + let partition = view.getUint32(0) + let sequence = Number(view.getBigUint64(8)) + assert.equal(sequence, delivered[partition]! + 1) + assert.equal(message.producer, producers[partition]) + assert.equal(message.offset, BigInt(sequence - 1)) + assert.equal(message.partitionSession.deref()?.partitionId, BigInt(partition)) + let expected = (partition + sequence) % 251 + for (let index = 4; index < data.length; index++) { + if (index >= 8 && index < 16) continue + if (data[index] !== expected) throw new Error(`Payload mismatch at byte ${index}`) + } + delivered[partition] = sequence + } + // oxlint-disable-next-line no-await-in-loop + await reader.commit(batch) + committed = [...delivered] + if (committed.every((count, partition) => count === accepted[partition])) break + } +} + +async function run() { + await using driver = new Driver(process.env['YDB_CONNECTION_STRING']!, { + 'ydb.sdk.enable_discovery': false, + }) + await driver.ready(signal.signal) + let service = driver.createClient(TopicServiceDefinition) + let result = await service.createTopic( + create(CreateTopicRequestSchema, { + path: topic, + partitioningSettings: { minActivePartitions: 3n, maxActivePartitions: 3n }, + consumers: [{ name: consumer }], + supportedCodecs: { codecs: [Codec.RAW, Codec.GZIP, Codec.ZSTD] }, + }), + { signal: signal.signal } + ) + assert.equal(result.operation?.status, StatusIds_StatusCode.SUCCESS) + let readerChannel = channel('ydb:topic.reader.reconnecting') + let writerChannel = channel('ydb:topic.writer.reconnecting') + let onReader = (event: unknown) => { + if ((event as { driver: unknown }).driver === driver.identity) reconnects.reader++ + } + let onWriter = (event: unknown) => { + if ((event as { driver: unknown }).driver === driver.identity) reconnects.writer++ + } + readerChannel.subscribe(onReader) + writerChannel.subscribe(onWriter) + using _ = { + [Symbol.dispose]() { + readerChannel.unsubscribe(onReader) + writerChannel.unsubscribe(onWriter) + }, + } + + let openClients = (): Clients => { + let clientGeneration = generation++ + let reader = createTopicReader(driver, { topic, consumer, maxBufferBytes: 256n * 1024n }) + let writers = codecs.map((codec, partition) => + createTopicWriter(driver, { + topic, + producer: producers[partition]!, + partitionId: BigInt(partition), + codec, + maxInflightCount: 16, + maxBufferBytes: 8n * 1024n * 1024n, + }) + ) + return { + reader, + writers, + async close(scenario, epoch) { + await reader.close() + await Promise.all(writers.map((writer) => writer.close(signal.signal))) + closedClients.push({ + ref: new WeakRef(reader), + role: 'reader', + generation: clientGeneration, + scenario, + epoch, + }) + for (let [partition, writer] of writers.entries()) { + closedClients.push({ + ref: new WeakRef(writer), + role: 'writer', + generation: clientGeneration, + partition, + scenario, + epoch, + }) + } + }, + destroy() { + reader.destroy() + for (let writer of writers) writer.destroy() + }, + } + } + let clients: Clients | undefined = openClients() + let enqueue = () => { + for (let partition = 0; partition < codecs.length; partition++) { + for (let index = 0; index < messagesPerEpoch; index++) { + let sequence = accepted[partition]! + 1 + let payload = createPayload(partition, sequence) + clients!.writers[partition]!.write(payload, { + metadataItems: { source: new Uint8Array([partition]) }, + }) + accepted[partition] = sequence + // Intentional leak control for validating this fixture's memory verdict. + if (process.env['YDB_MEMORY_RETAIN_PAYLOADS'] === '1') retained.push(payload) + } + } + } + try { + for (let scenario of ['steady', 'reconnect', 'replace']) { + for (let epoch = 0; epoch < epochs; epoch++) { + enqueue() + // oxlint-disable-next-line no-await-in-loop + await Promise.all( + clients!.writers.map(async (writer, partition) => { + assert.equal( + await writer.flush(signal.signal), + BigInt(accepted[partition]!) + ) + acknowledged[partition] = accepted[partition]! + }) + ) + samples.push({ + phase: 'loaded', + scenario, + epoch, + ...memorySample(), + }) + // oxlint-disable-next-line no-await-in-loop + await consumeEpoch(clients!.reader) + // oxlint-disable-next-line no-await-in-loop + await checkpoint('drained', epoch, scenario) + if (scenario === 'replace' && epoch + 1 < epochs) { + // oxlint-disable-next-line no-await-in-loop + await clients!.close(scenario, epoch) + clients = undefined + // oxlint-disable-next-line no-await-in-loop + await sleep(10, undefined, { signal: signal.signal }) + // oxlint-disable-next-line no-await-in-loop + await checkpoint('replaced', epoch, scenario) + clients = openClients() + } else if (scenario === 'reconnect' && epoch + 1 < epochs) { + let disconnected = once(process, 'message', { signal: signal.signal }) + process.send!({ type: 'disconnect' }) + // oxlint-disable-next-line no-await-in-loop + await disconnected + } + if (epochPauseMs > 0) { + // oxlint-disable-next-line no-await-in-loop + await sleep(epochPauseMs, undefined, { signal: signal.signal }) + } + } + } + await clients!.close('final', epochs - 1) + clients = undefined + } finally { + clients?.destroy() + await service.dropTopic(create(DropTopicRequestSchema, { path: topic }), { + signal: AbortSignal.timeout(5000), + }) + } + driver.close() + await sleep(25) + await checkpoint('closed', epochs) +} + +try { + await run() + await checkpoint('released', epochs) + let closedClientsRemaining = closedClients + .filter(({ ref }) => ref.deref() !== undefined) + .map((client) => { + let label: ClosedClientLabel = { + role: client.role, + generation: client.generation, + scenario: client.scenario, + epoch: client.epoch, + } + if (client.partition !== undefined) label.partition = client.partition + return label + }) + let report: MemoryReport = { + topic, + runtime, + node: process.version, + platform: process.platform, + arch: process.arch, + epochs, + epochPauseMs, + payloadBytes, + messagesPerEpoch, + accepted, + acknowledged, + delivered, + committed, + reconnects, + closedClientsAlive: closedClientsRemaining.length, + closedClientsRemaining, + retainedControlBytes: retained.reduce((bytes, payload) => bytes + payload.byteLength, 0), + samples, + } + await writeFile(process.env['MEMORY_REPORT_FILE']!, JSON.stringify(report)) +} catch (error) { + console.error(error) + process.exitCode = 1 +} finally { + process.disconnect?.() +} diff --git a/packages/topic/tests/reader-parity.test.ts b/packages/topic/tests/reader-parity.test.ts index fc618f88..cb43d287 100644 --- a/packages/topic/tests/reader-parity.test.ts +++ b/packages/topic/tests/reader-parity.test.ts @@ -10,6 +10,7 @@ import { TopicServiceDefinition, } from '@ydbjs/api/topic' import { Driver } from '@ydbjs/core' +import { Status } from 'nice-grpc' import type { TopicMessage } from '../src/message.js' import { type TopicReader, createTopicReader } from '../src/reader/index.js' @@ -316,3 +317,26 @@ test( expect(received.map((message) => message.offset)).toEqual([7n, 8n, 9n]) } ) + +test( + 'fails an oversized read response instead of reconnecting to the same message', + { timeout: 10_000 }, + async (tc) => { + let topic = await makeTopic('receive-limit') + await using writer = createTopicWriter(driver, { topic }) + writer.write(new Uint8Array(2 * 1024 * 1024)) + await writer.close(tc.signal) + + using limited = new Driver(inject('connectionString'), { + 'ydb.sdk.enable_discovery': false, + channelOptions: { 'grpc.max_receive_message_length': 1024 * 1024 }, + }) + await limited.ready(tc.signal) + using reader = createTopicReader(limited, { topic, consumer: consumerName }) + let iterator = reader.read({ signal: tc.signal })[Symbol.asyncIterator]() + await expect(iterator.next()).rejects.toMatchObject({ + code: Status.RESOURCE_EXHAUSTED, + details: expect.stringMatching(/larger than/i), + }) + } +) diff --git a/packages/topic/tests/reconnect.test.ts b/packages/topic/tests/reconnect.test.ts index ffe7d7e5..6286bf20 100644 --- a/packages/topic/tests/reconnect.test.ts +++ b/packages/topic/tests/reconnect.test.ts @@ -1,21 +1,6 @@ -// eslint-disable no-await-in-loop - -/** - * Integration tests for topic reader/writer stability across connection pool - * refreshes (issue #557). - * - * The reproduction scenario: the driver periodically closes all pool channels - * and replaces them with fresh ones after each discovery round. - * Active bidirectional streams (reader / writer) receive a CANCELLED gRPC - * status and must reconnect transparently. - * - * Before the fix: - * - CANCELLED was not treated as a retryable error for streams - * - writer became permanently destroyed after the first pool refresh - * - reader entered a zombie state and blocked read() forever - */ - -import { afterEach, beforeEach, expect, inject, test } from 'vitest' +import { channel } from 'node:diagnostics_channel' +import { once } from 'node:events' +import { type Socket, createConnection, createServer } from 'node:net' import { create } from '@bufbuild/protobuf' import { @@ -24,121 +9,166 @@ import { TopicServiceDefinition, } from '@ydbjs/api/topic' import { Driver } from '@ydbjs/core' +import { expect, inject, test } from 'vitest' -import { createTopicReader } from '../src/reader/index.js' -import { createTopicWriter } from '../src/writer/index.js' - -// Shared admin driver (no discovery) for topic lifecycle — avoids connection -// disruptions from pool refreshes during setup/teardown. -let adminDriver = new Driver(inject('connectionString'), { - 'ydb.sdk.enable_discovery': false, -}) -await adminDriver.ready() - -let topicService = adminDriver.createClient(TopicServiceDefinition) - -let testTopicName: string -let testConsumerName: string -let testDriver!: Driver - -beforeEach(async () => { - testTopicName = `test-topic-reconnect-${Date.now()}` - testConsumerName = `test-consumer-reconnect-${Date.now()}` - - await topicService.createTopic( - create(CreateTopicRequestSchema, { - path: testTopicName, - partitioningSettings: { - minActivePartitions: 1n, - maxActivePartitions: 1n, - }, - consumers: [{ name: testConsumerName }], - }) - ) - - // Discovery enabled with a 10 s interval: every round closes old pool - // channels and opens fresh ones, sending CANCELLED to active streams. - testDriver = new Driver(inject('connectionString'), { - 'ydb.sdk.enable_discovery': true, - 'ydb.sdk.discovery_interval_ms': 10_000, - 'ydb.sdk.discovery_timeout_ms': 5_000, - }) - await testDriver.ready() -}) - -afterEach(async () => { - testDriver.close() - - await topicService.dropTopic(create(DropTopicRequestSchema, { path: testTopicName })) -}) +import { createTopicReader } from '../src/reader/index.ts' +import { createTopicWriter } from '../src/writer/index.ts' test( - 'writer and reader survive repeated connection pool refreshes over 60 s', - { timeout: 90_000 }, - async () => { - let TEST_DURATION_MS = 60_000 - let WRITE_INTERVAL_MS = 500 // one message every 500 ms → ~120 messages - - let writtenSeqNos = new Set() - let receivedSeqNos = new Set() - let readerAc = new AbortController() - - await using writer = createTopicWriter(testDriver, { - topic: testTopicName, + 'reconciles writes and commits after TCP disconnects with server responses withheld', + { timeout: 30_000 }, + async (tc) => { + let upstream = new URL(inject('connectionString')) + let sockets = new Set() + let responses = new Set() + let requestBytes = 0 + let proxy = createServer((client) => { + let server = createConnection({ host: upstream.hostname, port: Number(upstream.port) }) + sockets.add(client) + sockets.add(server) + responses.add(server) + client.on('data', (data) => { + requestBytes += data.length + }) + client.on('error', () => server.destroy()) + server.on('error', () => client.destroy()) + client.on('close', () => { + sockets.delete(client) + server.destroy() + }) + server.on('close', () => { + sockets.delete(server) + responses.delete(server) + client.destroy() + }) + client.pipe(server).pipe(client) }) + let cut = () => { + let count = sockets.size + for (let socket of sockets) { + socket.destroy() + } + return count + } + await using fault = { + cut, + async [Symbol.asyncDispose]() { + cut() + await proxy[Symbol.asyncDispose]() + }, + } + let disconnectAfterRequest = async (previousBytes: number) => { + await expect.poll(() => requestBytes).toBeGreaterThan(previousBytes) + expect(fault.cut()).toBeGreaterThan(0) + } + proxy.listen(0, '127.0.0.1') + await once(proxy, 'listening', { signal: tc.signal }) + let address = proxy.address() + if (!address || typeof address === 'string') { + throw new Error('Expected TCP proxy address') + } - await using reader = createTopicReader(testDriver, { - topic: testTopicName, - consumer: testConsumerName, + using admin = new Driver(upstream.toString(), { 'ydb.sdk.enable_discovery': false }) + await admin.ready(tc.signal) + let service = admin.createClient(TopicServiceDefinition) + await using topic = { + path: `topic-tcp-reconnect-${Date.now()}`, + async [Symbol.asyncDispose]() { + await service.dropTopic(create(DropTopicRequestSchema, { path: topic.path })) + }, + } + await service.createTopic( + create(CreateTopicRequestSchema, { + path: topic.path, + partitioningSettings: { minActivePartitions: 1n, maxActivePartitions: 1n }, + consumers: [{ name: 'consumer' }], + }), + { signal: tc.signal } + ) + + using driver = new Driver(`grpc://127.0.0.1:${address.port}${upstream.pathname}`, { + 'ydb.sdk.enable_discovery': false, }) - - // ── Reader loop ──────────────────────────────────────────────────────── - let readerTask = (async () => { - for await (let batch of reader.read({ - batchWindowMs: 5_000, - signal: readerAc.signal, - })) { - for (let msg of batch) { - receivedSeqNos.add(msg.seqNo) - } - - if (batch.length > 0) { + await driver.ready(tc.signal) + let readerEvents = channel('ydb:topic.reader.reconnecting') + let writerEvents = channel('ydb:topic.writer.reconnecting') + let onReaderReconnect = (event: unknown) => { + if ((event as { driver: unknown }).driver === driver.identity) { + reconnects.read++ + } + } + let onWriterReconnect = (event: unknown) => { + if ((event as { driver: unknown }).driver === driver.identity) { + reconnects.write++ + } + } + readerEvents.subscribe(onReaderReconnect) + writerEvents.subscribe(onWriterReconnect) + using reconnects = { + read: 0, + write: 0, + [Symbol.dispose]() { + readerEvents.unsubscribe(onReaderReconnect) + writerEvents.unsubscribe(onWriterReconnect) + }, + } + await using writer = createTopicWriter(driver, { topic: topic.path, producer: 'producer' }) + await using reader = createTopicReader(driver, { topic: topic.path, consumer: 'consumer' }) + let committed = new Set() + let commitInterrupted = false + let readTask = (async () => { + for await (let batch of reader.read({ signal: tc.signal })) { + if (!commitInterrupted && batch.some((message) => message.payload[0]! >= 20)) { + commitInterrupted = true + for (let server of responses) { + server.pause() + } + let sentBefore = requestBytes + let pending = reader.commit(batch) + void pending.catch(() => {}) + await disconnectAfterRequest(sentBefore) + await pending + } else { await reader.commit(batch) } + for (let message of batch) { + committed.add(message.payload[0]!) + } + if (committed.size === 30) { + break + } } })() - - // ── Writer loop ──────────────────────────────────────────────────────── - let deadline = Date.now() + TEST_DURATION_MS - let index = 0 - - while (Date.now() < deadline) { - writer.write(new TextEncoder().encode(`msg-${index++}`)) - - // write() is void; flush() waits for the server ACK and returns the - // last committed seqNo (one write per iteration ⇒ this message's seqNo). - let seqNo = await writer.flush() - writtenSeqNos.add(seqNo) - - let remaining = deadline - Date.now() - if (remaining > 0) { - await new Promise((r) => setTimeout(r, Math.min(WRITE_INTERVAL_MS, remaining))) + // Await the task below; attach its rejection handler before producing traffic. + void readTask.catch(() => {}) + + for (let round = 0; round < 3; round++) { + let sentBefore = requestBytes + if (round === 1) { + for (let server of responses) { + server.pause() + } } + for (let index = round * 10; index < (round + 1) * 10; index++) { + writer.write(new Uint8Array([index])) + } + let flushed = writer.flush(tc.signal) + void flushed.catch(() => {}) + if (round === 1) { + // oxlint-disable-next-line no-await-in-loop + await disconnectAfterRequest(sentBefore) + } + // oxlint-disable-next-line no-await-in-loop + await expect(flushed).resolves.toBe(BigInt((round + 1) * 10)) + // oxlint-disable-next-line no-await-in-loop + await expect.poll(() => committed.size).toBe((round + 1) * 10) } - - // Give the reader up to 10 s to drain any in-flight messages. - let drainDeadline = Date.now() + 10_000 - while (receivedSeqNos.size < writtenSeqNos.size && Date.now() < drainDeadline) { - await new Promise((r) => setTimeout(r, 200)) - } - - readerAc.abort() - await readerTask.catch(() => {}) - - // ── Assertions ───────────────────────────────────────────────────────── - expect(writtenSeqNos.size).toBeGreaterThan(0) - - let missing = [...writtenSeqNos].filter((seqNo) => !receivedSeqNos.has(seqNo)) - expect(missing, `missing seqNos: ${missing.join(', ')}`).toHaveLength(0) + await readTask + expect(commitInterrupted).toBe(true) + expect([...committed].sort((a, b) => a - b)).toEqual( + Array.from({ length: 30 }, (_, index) => index) + ) + expect(reconnects.read).toBeGreaterThanOrEqual(2) + expect(reconnects.write).toBeGreaterThanOrEqual(2) } ) diff --git a/packages/topic/tests/writer-protocol.test.ts b/packages/topic/tests/writer-protocol.test.ts index 3ab0153e..d3b79a52 100644 --- a/packages/topic/tests/writer-protocol.test.ts +++ b/packages/topic/tests/writer-protocol.test.ts @@ -21,7 +21,7 @@ import { AsyncPriorityQueue } from '@ydbjs/fsm/queue' import { ClientError, Status } from 'nice-grpc' import { createTopicWriter } from '../src/writer/index.js' -import { isRetryableWriterError } from '../src/writer/writer-state.js' +import { isRetryableTopicError } from '../src/retry.js' // Real-YDB protocol truth for the writer FSM migration. The reconnect dedup design // (drop in-flight messages the server already persisted, walk acks as an ordered @@ -240,7 +240,7 @@ test('rejects an oversized payload with a permanent error', async (tc) => { expect(error).toBeInstanceOf(ClientError) expect((error as ClientError).code).toBe(Status.RESOURCE_EXHAUSTED) expect((error as ClientError).details).toMatch(/Received message larger than max/) - expect(isRetryableWriterError(error)).toBe(false) + expect(isRetryableTopicError(error)).toBe(false) } // Client-side rejection: over the 64MiB client gRPC send cap. @@ -262,6 +262,6 @@ test('rejects an oversized payload with a permanent error', async (tc) => { expect((error as ClientError).details).toMatch( /Attempted to send message with a size larger than/ ) - expect(isRetryableWriterError(error)).toBe(false) + expect(isRetryableTopicError(error)).toBe(false) } }) diff --git a/packages/topic/tests/writer.test.ts b/packages/topic/tests/writer.test.ts index 3d643885..231b8a2f 100644 --- a/packages/topic/tests/writer.test.ts +++ b/packages/topic/tests/writer.test.ts @@ -1,3 +1,6 @@ +import { createHash } from 'node:crypto' +import { setTimeout as sleep } from 'node:timers/promises' + import { create } from '@bufbuild/protobuf' import { CreateTopicRequestSchema, @@ -80,6 +83,36 @@ test('writes messages and reads them back in order', async () => { expect(seqNos).toEqual([1n, 2n, 3n]) }) +test('publishes isolated writes across idle intervals without an explicit flush', async (tc) => { + let flushIntervalMs = 20 + await using writer = createTopicWriter(driver, { + topic: testTopicName, + producer: testProducerName, + flushIntervalMs, + }) + await using reader = createTopicReader(driver, { + topic: testTopicName, + consumer: testConsumerName, + }) + let messages = reader.read({ limit: 1, signal: tc.signal })[Symbol.asyncIterator]() + try { + writer.write(encode('first')) + let first = await messages.next() + expect(first.done).toBe(false) + expect(new TextDecoder().decode(first.value![0]!.payload)).toBe('first') + await reader.commit(first.value!) + + await sleep(3 * flushIntervalMs, undefined, { signal: tc.signal }) + writer.write(encode('after idle')) + let second = await messages.next() + expect(second.done).toBe(false) + expect(new TextDecoder().decode(second.value![0]!.payload)).toBe('after idle') + await reader.commit(second.value!) + } finally { + await messages.return?.() + } +}) + test( 'batches concurrent per-message commits behind one server acknowledgment', { timeout: 30_000 }, @@ -323,3 +356,42 @@ test('writes many messages preserving order', async () => { expect(received).toHaveLength(total) expect(received).toEqual(Array.from({ length: total }, (_, i) => `msg-${i}`)) }) + +test.for([1, 6])( + 'round-trips 48 MiB of RAW payload in %i messages at the buffer limit', + async (count, tc) => { + let totalBytes = 48 * 1024 * 1024 + let payload = new Uint8Array(totalBytes / count).fill(0x5a) + let digest = createHash('sha256').update(payload).digest('hex') + let metadataItems = { trace: new Uint8Array(256).fill(0x31) } + await using writer = createTopicWriter(driver, { + topic: testTopicName, + producer: testProducerName, + maxBufferBytes: BigInt(totalBytes), + }) + for (let i = 0; i < count; i++) writer.write(payload, { metadataItems }) + expect(() => writer.write(new Uint8Array(1))).toThrow(/buffer is full/) + expect(await writer.flush(tc.signal)).toBe(BigInt(count)) + await using reader = createTopicReader(driver, { + topic: testTopicName, + consumer: testConsumerName, + }) + let received = 0 + for await (let batch of reader.read({ + limit: count, + batchWindowMs: 100, + signal: tc.signal, + })) { + for (let message of batch) { + expect(message.payload.byteLength).toBe(payload.byteLength) + expect(createHash('sha256').update(message.payload).digest('hex')).toBe(digest) + expect(Object.keys(message.metadataItems ?? {})).toEqual(['trace']) + expect(new Uint8Array(message.metadataItems!.trace!)).toEqual(metadataItems.trace) + expect(message.seqNo).toBe(BigInt(++received)) + } + await reader.commit(batch) + if (received === count) break + } + expect(received).toBe(count) + } +) diff --git a/packages/topic/tsconfig.json b/packages/topic/tsconfig.json index 98c320f7..0e55a8a5 100644 --- a/packages/topic/tsconfig.json +++ b/packages/topic/tsconfig.json @@ -4,5 +4,5 @@ "rootDir": "./src", "outDir": "./dist" }, - "exclude": ["dist/**/*", "**/*.test.ts", "**/*.bench.ts", "vitest.config.ts"] + "exclude": ["dist/**/*", "tests/**/*", "**/*.test.ts", "**/*.bench.ts", "vitest.config.ts"] }