From 1c39007cf9426764fe142389fec0599545faa56d Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 3 Nov 2025 14:29:53 -0800 Subject: [PATCH 01/26] hacking --- package-lock.json | 3371 +++++++++++++++++++++++ package.json | 4 + src/accessTokens.ts | 72 +- src/basin.ts | 22 +- src/basins.ts | 144 +- src/common.ts | 39 + src/error.ts | 68 + src/generated/client/types.gen.ts | 3 +- src/index.ts | 6 + src/lib/retry.ts | 113 + src/lib/stream/transport/fetch/index.ts | 4 + src/lib/stream/transport/s2s/index.ts | 36 + src/lib/stream/types.ts | 4 + src/metrics.ts | 76 +- src/s2.ts | 22 +- src/stream.ts | 66 +- src/streams.ts | 142 +- src/tests/appendSession.e2e.test.ts | 14 +- src/tests/retry.test.ts | 103 + 19 files changed, 4048 insertions(+), 261 deletions(-) create mode 100644 package-lock.json create mode 100644 src/lib/retry.ts create mode 100644 src/tests/retry.test.ts diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e02494d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,3371 @@ +{ + "name": "@s2-dev/streamstore", + "version": "0.17.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@s2-dev/streamstore", + "version": "0.17.0", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.2", + "@biomejs/biome": "2.2.5", + "@changesets/cli": "^2.29.7", + "@hey-api/openapi-ts": "^0.86.0", + "@protobuf-ts/plugin": "^2.11.1", + "@types/bun": "^1.3.1", + "@types/debug": "^4.1.12", + "openapi-typescript": "^7.10.1", + "protoc": "^33.0.0", + "typedoc": "^0.28.14", + "vitest": "^4.0.2" + }, + "peerDependencies": { + "typescript": "^5.9.3" + } + }, + "node_modules/@andrewbranch/untar.js": { + "version": "1.0.3", + "dev": true + }, + "node_modules/@arethetypeswrong/cli": { + "version": "0.18.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@arethetypeswrong/core": "0.18.2", + "chalk": "^4.1.2", + "cli-table3": "^0.6.3", + "commander": "^10.0.1", + "marked": "^9.1.2", + "marked-terminal": "^7.1.0", + "semver": "^7.5.4" + }, + "bin": { + "attw": "dist/index.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@arethetypeswrong/core": { + "version": "0.18.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@andrewbranch/untar.js": "^1.0.3", + "@loaderkit/resolve": "^1.0.2", + "cjs-module-lexer": "^1.2.3", + "fflate": "^0.8.2", + "lru-cache": "^11.0.1", + "semver": "^7.5.4", + "typescript": "5.6.1-rc", + "validate-npm-package-name": "^5.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@arethetypeswrong/core/node_modules/typescript": { + "version": "5.6.1-rc", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.28.4", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@biomejs/biome": { + "version": "2.2.5", + "dev": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "2.2.5", + "@biomejs/cli-darwin-x64": "2.2.5", + "@biomejs/cli-linux-arm64": "2.2.5", + "@biomejs/cli-linux-arm64-musl": "2.2.5", + "@biomejs/cli-linux-x64": "2.2.5", + "@biomejs/cli-linux-x64-musl": "2.2.5", + "@biomejs/cli-win32-arm64": "2.2.5", + "@biomejs/cli-win32-x64": "2.2.5" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "2.2.5", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@braidai/lang": { + "version": "1.1.2", + "dev": true, + "license": "ISC" + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.10.0", + "dev": true, + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@bufbuild/protoplugin": { + "version": "2.10.0", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "2.10.0", + "@typescript/vfs": "^1.5.2", + "typescript": "5.4.5" + } + }, + "node_modules/@bufbuild/protoplugin/node_modules/typescript": { + "version": "5.4.5", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/@changesets/apply-release-plan": { + "version": "7.0.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/config": "^3.1.1", + "@changesets/get-version-range-type": "^0.4.0", + "@changesets/git": "^3.0.4", + "@changesets/should-skip-package": "^0.1.2", + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3", + "detect-indent": "^6.0.0", + "fs-extra": "^7.0.1", + "lodash.startcase": "^4.4.0", + "outdent": "^0.5.0", + "prettier": "^2.7.1", + "resolve-from": "^5.0.0", + "semver": "^7.5.3" + } + }, + "node_modules/@changesets/assemble-release-plan": { + "version": "6.0.9", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/errors": "^0.2.0", + "@changesets/get-dependents-graph": "^2.1.3", + "@changesets/should-skip-package": "^0.1.2", + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3", + "semver": "^7.5.3" + } + }, + "node_modules/@changesets/changelog-git": { + "version": "0.2.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/types": "^6.1.0" + } + }, + "node_modules/@changesets/cli": { + "version": "2.29.7", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/apply-release-plan": "^7.0.13", + "@changesets/assemble-release-plan": "^6.0.9", + "@changesets/changelog-git": "^0.2.1", + "@changesets/config": "^3.1.1", + "@changesets/errors": "^0.2.0", + "@changesets/get-dependents-graph": "^2.1.3", + "@changesets/get-release-plan": "^4.0.13", + "@changesets/git": "^3.0.4", + "@changesets/logger": "^0.1.1", + "@changesets/pre": "^2.0.2", + "@changesets/read": "^0.6.5", + "@changesets/should-skip-package": "^0.1.2", + "@changesets/types": "^6.1.0", + "@changesets/write": "^0.4.0", + "@inquirer/external-editor": "^1.0.0", + "@manypkg/get-packages": "^1.1.3", + "ansi-colors": "^4.1.3", + "ci-info": "^3.7.0", + "enquirer": "^2.4.1", + "fs-extra": "^7.0.1", + "mri": "^1.2.0", + "p-limit": "^2.2.0", + "package-manager-detector": "^0.2.0", + "picocolors": "^1.1.0", + "resolve-from": "^5.0.0", + "semver": "^7.5.3", + "spawndamnit": "^3.0.1", + "term-size": "^2.1.0" + }, + "bin": { + "changeset": "bin.js" + } + }, + "node_modules/@changesets/config": { + "version": "3.1.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/errors": "^0.2.0", + "@changesets/get-dependents-graph": "^2.1.3", + "@changesets/logger": "^0.1.1", + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3", + "fs-extra": "^7.0.1", + "micromatch": "^4.0.8" + } + }, + "node_modules/@changesets/errors": { + "version": "0.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "extendable-error": "^0.1.5" + } + }, + "node_modules/@changesets/get-dependents-graph": { + "version": "2.1.3", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3", + "picocolors": "^1.1.0", + "semver": "^7.5.3" + } + }, + "node_modules/@changesets/get-release-plan": { + "version": "4.0.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/assemble-release-plan": "^6.0.9", + "@changesets/config": "^3.1.1", + "@changesets/pre": "^2.0.2", + "@changesets/read": "^0.6.5", + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3" + } + }, + "node_modules/@changesets/get-version-range-type": { + "version": "0.4.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@changesets/git": { + "version": "3.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/errors": "^0.2.0", + "@manypkg/get-packages": "^1.1.3", + "is-subdir": "^1.1.1", + "micromatch": "^4.0.8", + "spawndamnit": "^3.0.1" + } + }, + "node_modules/@changesets/logger": { + "version": "0.1.1", + "dev": true, + "license": "MIT", + "dependencies": { + "picocolors": "^1.1.0" + } + }, + "node_modules/@changesets/parse": { + "version": "0.4.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/types": "^6.1.0", + "js-yaml": "^3.13.1" + } + }, + "node_modules/@changesets/parse/node_modules/js-yaml": { + "version": "3.14.1", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@changesets/parse/node_modules/js-yaml/node_modules/argparse": { + "version": "1.0.10", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@changesets/pre": { + "version": "2.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/errors": "^0.2.0", + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3", + "fs-extra": "^7.0.1" + } + }, + "node_modules/@changesets/read": { + "version": "0.6.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/git": "^3.0.4", + "@changesets/logger": "^0.1.1", + "@changesets/parse": "^0.4.1", + "@changesets/types": "^6.1.0", + "fs-extra": "^7.0.1", + "p-filter": "^2.1.0", + "picocolors": "^1.1.0" + } + }, + "node_modules/@changesets/should-skip-package": { + "version": "0.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/types": "^6.1.0", + "@manypkg/get-packages": "^1.1.3" + } + }, + "node_modules/@changesets/types": { + "version": "6.1.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@changesets/write": { + "version": "0.4.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@changesets/types": "^6.1.0", + "fs-extra": "^7.0.1", + "human-id": "^4.1.1", + "prettier": "^2.7.1" + } + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.11", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@gerrit0/mini-shiki": { + "version": "3.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/engine-oniguruma": "^3.14.0", + "@shikijs/langs": "^3.14.0", + "@shikijs/themes": "^3.14.0", + "@shikijs/types": "^3.14.0", + "@shikijs/vscode-textmate": "^10.0.2" + } + }, + "node_modules/@hey-api/codegen-core": { + "version": "0.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@hey-api/json-schema-ref-parser": { + "version": "1.2.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.15", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + } + }, + "node_modules/@hey-api/openapi-ts": { + "version": "0.86.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@hey-api/codegen-core": "^0.3.1", + "@hey-api/json-schema-ref-parser": "1.2.1", + "ansi-colors": "4.1.3", + "c12": "3.3.1", + "color-support": "1.1.3", + "commander": "14.0.1", + "handlebars": "4.7.8", + "open": "10.2.0", + "semver": "7.7.2" + }, + "bin": { + "openapi-ts": "bin/run.js" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/hey-api" + }, + "peerDependencies": { + "typescript": ">=5.5.3" + } + }, + "node_modules/@hey-api/openapi-ts/node_modules/commander": { + "version": "14.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/@hey-api/openapi-ts/node_modules/semver": { + "version": "7.7.2", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "chardet": "^2.1.0", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "dev": true, + "license": "MIT" + }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/@loaderkit/resolve": { + "version": "1.0.4", + "dev": true, + "license": "ISC", + "dependencies": { + "@braidai/lang": "^1.0.0" + } + }, + "node_modules/@manypkg/find-root": { + "version": "1.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.5.5", + "@types/node": "^12.7.1", + "find-up": "^4.1.0", + "fs-extra": "^8.1.0" + } + }, + "node_modules/@manypkg/find-root/node_modules/@types/node": { + "version": "12.20.55", + "dev": true, + "license": "MIT" + }, + "node_modules/@manypkg/find-root/node_modules/fs-extra": { + "version": "8.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/@manypkg/get-packages": { + "version": "1.1.3", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.5.5", + "@changesets/types": "^4.0.1", + "@manypkg/find-root": "^1.1.0", + "fs-extra": "^8.1.0", + "globby": "^11.0.0", + "read-yaml-file": "^1.1.0" + } + }, + "node_modules/@manypkg/get-packages/node_modules/@changesets/types": { + "version": "4.1.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@manypkg/get-packages/node_modules/fs-extra": { + "version": "8.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@protobuf-ts/plugin": { + "version": "2.11.1", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^2.4.0", + "@bufbuild/protoplugin": "^2.4.0", + "@protobuf-ts/protoc": "^2.11.1", + "@protobuf-ts/runtime": "^2.11.1", + "@protobuf-ts/runtime-rpc": "^2.11.1", + "typescript": "^3.9" + }, + "bin": { + "protoc-gen-dump": "bin/protoc-gen-dump", + "protoc-gen-ts": "bin/protoc-gen-ts" + } + }, + "node_modules/@protobuf-ts/plugin/node_modules/typescript": { + "version": "3.9.10", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/@protobuf-ts/protoc": { + "version": "2.11.1", + "dev": true, + "license": "Apache-2.0", + "bin": { + "protoc": "protoc.js" + } + }, + "node_modules/@protobuf-ts/runtime": { + "version": "2.11.1", + "dev": true, + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@protobuf-ts/runtime-rpc": { + "version": "2.11.1", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@protobuf-ts/runtime": "^2.11.1" + } + }, + "node_modules/@redocly/ajv": { + "version": "8.11.3", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js-replace": "^1.0.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@redocly/config": { + "version": "0.22.2", + "dev": true, + "license": "MIT" + }, + "node_modules/@redocly/openapi-core": { + "version": "1.34.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/ajv": "^8.11.2", + "@redocly/config": "^0.22.0", + "colorette": "^1.2.0", + "https-proxy-agent": "^7.0.5", + "js-levenshtein": "^1.1.6", + "js-yaml": "^4.1.0", + "minimatch": "^5.0.1", + "pluralize": "^8.0.0", + "yaml-ast-parser": "0.0.43" + }, + "engines": { + "node": ">=18.17.0", + "npm": ">=9.5.0" + } + }, + "node_modules/@redocly/openapi-core/node_modules/minimatch": { + "version": "5.1.6", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.52.5", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "3.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.14.0", + "@shikijs/vscode-textmate": "^10.0.2" + } + }, + "node_modules/@shikijs/langs": { + "version": "3.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.14.0" + } + }, + "node_modules/@shikijs/themes": { + "version": "3.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "3.14.0" + } + }, + "node_modules/@shikijs/types": { + "version": "3.14.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/bun": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.3.1" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.9.2", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.2", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/@typescript/vfs": { + "version": "1.6.2", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + }, + "peerDependencies": { + "typescript": "*" + } + }, + "node_modules/@vitest/expect": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.0.4", + "@vitest/utils": "4.0.4", + "chai": "^6.0.1", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.0.4", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.19" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.0.4", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.4", + "magic-string": "^0.30.19", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.4", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "7.1.1", + "dev": true, + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "dev": true, + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/array-union": { + "version": "2.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/better-path-resolve": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "is-windows": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/bun-types": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + }, + "peerDependencies": { + "@types/react": "^19" + } + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/c12": { + "version": "3.3.1", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^4.0.3", + "confbox": "^0.2.2", + "defu": "^6.1.4", + "dotenv": "^17.2.3", + "exsolve": "^1.0.7", + "giget": "^2.0.0", + "jiti": "^2.6.1", + "ohash": "^2.0.11", + "pathe": "^2.0.3", + "perfect-debounce": "^2.0.0", + "pkg-types": "^2.3.0", + "rc9": "^2.1.2" + }, + "peerDependencies": { + "magicast": "^0.3.5" + }, + "peerDependenciesMeta": { + "magicast": { + "optional": true + } + } + }, + "node_modules/chai": { + "version": "6.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/change-case": { + "version": "5.4.4", + "dev": true, + "license": "MIT" + }, + "node_modules/char-regex": { + "version": "1.0.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/chardet": { + "version": "2.1.0", + "dev": true, + "license": "MIT" + }, + "node_modules/chokidar": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/citty": { + "version": "0.1.6", + "dev": true, + "license": "MIT", + "dependencies": { + "consola": "^3.2.3" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-highlight": { + "version": "2.1.11", + "dev": true, + "license": "ISC", + "dependencies": { + "chalk": "^4.0.0", + "highlight.js": "^10.7.1", + "mz": "^2.4.0", + "parse5": "^5.1.1", + "parse5-htmlparser2-tree-adapter": "^6.0.0", + "yargs": "^16.0.0" + }, + "bin": { + "highlight": "bin/highlight" + }, + "engines": { + "node": ">=8.0.0", + "npm": ">=5.0.0" + } + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "dev": true, + "license": "MIT" + }, + "node_modules/color-support": { + "version": "1.1.3", + "dev": true, + "license": "ISC", + "bin": { + "color-support": "bin.js" + } + }, + "node_modules/colorette": { + "version": "1.4.0", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "10.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/confbox": { + "version": "0.2.2", + "dev": true, + "license": "MIT" + }, + "node_modules/consola": { + "version": "3.4.2", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/default-browser": { + "version": "5.2.1", + "dev": true, + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defu": { + "version": "6.1.4", + "dev": true, + "license": "MIT" + }, + "node_modules/destr": { + "version": "2.0.5", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-indent": { + "version": "6.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/emojilib": { + "version": "2.4.0", + "dev": true, + "license": "MIT" + }, + "node_modules/enquirer": { + "version": "2.4.1", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/environment": { + "version": "1.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.25.11", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.11", + "@esbuild/android-arm": "0.25.11", + "@esbuild/android-arm64": "0.25.11", + "@esbuild/android-x64": "0.25.11", + "@esbuild/darwin-arm64": "0.25.11", + "@esbuild/darwin-x64": "0.25.11", + "@esbuild/freebsd-arm64": "0.25.11", + "@esbuild/freebsd-x64": "0.25.11", + "@esbuild/linux-arm": "0.25.11", + "@esbuild/linux-arm64": "0.25.11", + "@esbuild/linux-ia32": "0.25.11", + "@esbuild/linux-loong64": "0.25.11", + "@esbuild/linux-mips64el": "0.25.11", + "@esbuild/linux-ppc64": "0.25.11", + "@esbuild/linux-riscv64": "0.25.11", + "@esbuild/linux-s390x": "0.25.11", + "@esbuild/linux-x64": "0.25.11", + "@esbuild/netbsd-arm64": "0.25.11", + "@esbuild/netbsd-x64": "0.25.11", + "@esbuild/openbsd-arm64": "0.25.11", + "@esbuild/openbsd-x64": "0.25.11", + "@esbuild/openharmony-arm64": "0.25.11", + "@esbuild/sunos-x64": "0.25.11", + "@esbuild/win32-arm64": "0.25.11", + "@esbuild/win32-ia32": "0.25.11", + "@esbuild/win32-x64": "0.25.11" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.2.2", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/exsolve": { + "version": "1.0.7", + "dev": true, + "license": "MIT" + }, + "node_modules/extendable-error": { + "version": "0.1.7", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.19.1", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fflate": { + "version": "0.8.2", + "dev": true, + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs-extra": { + "version": "7.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/giget": { + "version": "2.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.0", + "defu": "^6.1.4", + "node-fetch-native": "^1.6.6", + "nypm": "^0.6.0", + "pathe": "^2.0.3" + }, + "bin": { + "giget": "dist/cli.mjs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "dev": true, + "license": "ISC" + }, + "node_modules/handlebars": { + "version": "4.7.8", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/highlight.js": { + "version": "10.7.3", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/human-id": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "bin": { + "human-id": "dist/cli.js" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/index-to-position": { + "version": "1.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-docker": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-subdir": { + "version": "1.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "better-path-resolve": "1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/is-windows": { + "version": "1.0.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-wsl": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.6.1", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-levenshtein": { + "version": "1.1.6", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/linkify-it": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.startcase": { + "version": "4.4.0", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "11.2.2", + "dev": true, + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/lunr": { + "version": "2.3.9", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/markdown-it": { + "version": "14.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/marked": { + "version": "9.1.6", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/marked-terminal": { + "version": "7.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "ansi-regex": "^6.1.0", + "chalk": "^5.4.1", + "cli-highlight": "^2.1.11", + "cli-table3": "^0.6.5", + "node-emoji": "^2.2.0", + "supports-hyperlinks": "^3.1.0" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "marked": ">=1 <16" + } + }, + "node_modules/marked-terminal/node_modules/chalk": { + "version": "5.6.2", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/mdurl": { + "version": "2.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/micromatch/node_modules/picomatch": { + "version": "2.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "license": "MIT" + }, + "node_modules/mz": { + "version": "2.7.0", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "dev": true, + "license": "MIT" + }, + "node_modules/node-emoji": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.6.0", + "char-regex": "^1.0.2", + "emojilib": "^2.4.0", + "skin-tone": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "dev": true, + "license": "MIT" + }, + "node_modules/nypm": { + "version": "0.6.2", + "dev": true, + "license": "MIT", + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.2", + "pathe": "^2.0.3", + "pkg-types": "^2.3.0", + "tinyexec": "^1.0.1" + }, + "bin": { + "nypm": "dist/cli.mjs" + }, + "engines": { + "node": "^14.16.0 || >=16.10.0" + } + }, + "node_modules/nypm/node_modules/tinyexec": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/object-assign": { + "version": "4.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ohash": { + "version": "2.0.11", + "dev": true, + "license": "MIT" + }, + "node_modules/open": { + "version": "10.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "wsl-utils": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openapi-typescript": { + "version": "7.10.1", + "dev": true, + "license": "MIT", + "dependencies": { + "@redocly/openapi-core": "^1.34.5", + "ansi-colors": "^4.1.3", + "change-case": "^5.4.4", + "parse-json": "^8.3.0", + "supports-color": "^10.2.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "openapi-typescript": "bin/cli.js" + }, + "peerDependencies": { + "typescript": "^5.x" + } + }, + "node_modules/outdent": { + "version": "0.5.0", + "dev": true, + "license": "MIT" + }, + "node_modules/p-filter": { + "version": "2.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-map": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-map": { + "version": "2.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/package-manager-detector": { + "version": "0.2.11", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^0.2.7" + } + }, + "node_modules/parse-json": { + "version": "8.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "index-to-position": "^1.1.0", + "type-fest": "^4.39.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "5.1.1", + "dev": true, + "license": "MIT" + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "6.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "parse5": "^6.0.1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { + "version": "6.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/perfect-debounce": { + "version": "2.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "4.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-types": { + "version": "2.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "confbox": "^0.2.2", + "exsolve": "^1.0.7", + "pathe": "^2.0.3" + } + }, + "node_modules/pluralize": { + "version": "8.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prettier": { + "version": "2.8.8", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/protoc": { + "version": "33.0.0", + "dev": true, + "license": "Apache-2.0", + "bin": { + "protoc": "protoc.cjs" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/rc9": { + "version": "2.1.2", + "dev": true, + "license": "MIT", + "dependencies": { + "defu": "^6.1.4", + "destr": "^2.0.3" + } + }, + "node_modules/read-yaml-file": { + "version": "1.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.5", + "js-yaml": "^3.6.1", + "pify": "^4.0.1", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-yaml-file/node_modules/js-yaml": { + "version": "3.14.1", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/read-yaml-file/node_modules/js-yaml/node_modules/argparse": { + "version": "1.0.10", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.52.5", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.52.5", + "@rollup/rollup-android-arm64": "4.52.5", + "@rollup/rollup-darwin-arm64": "4.52.5", + "@rollup/rollup-darwin-x64": "4.52.5", + "@rollup/rollup-freebsd-arm64": "4.52.5", + "@rollup/rollup-freebsd-x64": "4.52.5", + "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", + "@rollup/rollup-linux-arm-musleabihf": "4.52.5", + "@rollup/rollup-linux-arm64-gnu": "4.52.5", + "@rollup/rollup-linux-arm64-musl": "4.52.5", + "@rollup/rollup-linux-loong64-gnu": "4.52.5", + "@rollup/rollup-linux-ppc64-gnu": "4.52.5", + "@rollup/rollup-linux-riscv64-gnu": "4.52.5", + "@rollup/rollup-linux-riscv64-musl": "4.52.5", + "@rollup/rollup-linux-s390x-gnu": "4.52.5", + "@rollup/rollup-linux-x64-gnu": "4.52.5", + "@rollup/rollup-linux-x64-musl": "4.52.5", + "@rollup/rollup-openharmony-arm64": "4.52.5", + "@rollup/rollup-win32-arm64-msvc": "4.52.5", + "@rollup/rollup-win32-ia32-msvc": "4.52.5", + "@rollup/rollup-win32-x64-gnu": "4.52.5", + "@rollup/rollup-win32-x64-msvc": "4.52.5", + "fsevents": "~2.3.2" + } + }, + "node_modules/run-applescript": { + "version": "7.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.7.3", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/skin-tone": { + "version": "2.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "unicode-emoji-modifier-base": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/slash": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/spawndamnit": { + "version": "3.0.1", + "dev": true, + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "cross-spawn": "^7.0.5", + "signal-exit": "^4.0.1" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stackback": { + "version": "0.0.2", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width": { + "version": "4.2.3", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/supports-hyperlinks": { + "version": "3.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">=14.18" + }, + "funding": { + "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" + } + }, + "node_modules/supports-hyperlinks/node_modules/supports-color": { + "version": "7.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/term-size": { + "version": "2.2.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "dev": true, + "license": "MIT", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.0.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typedoc": { + "version": "0.28.14", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@gerrit0/mini-shiki": "^3.12.0", + "lunr": "^2.3.9", + "markdown-it": "^14.1.0", + "minimatch": "^9.0.5", + "yaml": "^2.8.1" + }, + "bin": { + "typedoc": "bin/typedoc" + }, + "engines": { + "node": ">= 18", + "pnpm": ">= 10" + }, + "peerDependencies": { + "typescript": "5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x || 5.5.x || 5.6.x || 5.7.x || 5.8.x || 5.9.x" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uc.micro": { + "version": "2.1.0", + "dev": true, + "license": "MIT" + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "dev": true, + "license": "MIT" + }, + "node_modules/unicode-emoji-modifier-base": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/uri-js-replace": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/validate-npm-package-name": { + "version": "5.0.1", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/vite": { + "version": "7.1.12", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.0.4", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.0.4", + "@vitest/mocker": "4.0.4", + "@vitest/pretty-format": "4.0.4", + "@vitest/runner": "4.0.4", + "@vitest/snapshot": "4.0.4", + "@vitest/spy": "4.0.4", + "@vitest/utils": "4.0.4", + "debug": "^4.4.3", + "es-module-lexer": "^1.7.0", + "expect-type": "^1.2.2", + "magic-string": "^0.30.19", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.0.4", + "@vitest/browser-preview": "4.0.4", + "@vitest/browser-webdriverio": "4.0.4", + "@vitest/ui": "4.0.4", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/which": { + "version": "2.0.2", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wsl-utils": { + "version": "0.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.8.1", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, + "node_modules/yaml-ast-parser": { + "version": "0.0.43", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/yargs": { + "version": "16.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "20.2.9", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + } + } +} diff --git a/package.json b/package.json index 4181455..a18bfe9 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "@hey-api/openapi-ts": "^0.86.0", "@protobuf-ts/plugin": "^2.11.1", "@types/bun": "^1.3.1", + "@types/debug": "^4.1.12", "openapi-typescript": "^7.10.1", "protoc": "^33.0.0", "typedoc": "^0.28.14", @@ -61,5 +62,8 @@ }, "peerDependencies": { "typescript": "^5.9.3" + }, + "dependencies": { + "debug": "^4.4.3" } } diff --git a/src/accessTokens.ts b/src/accessTokens.ts index 8c75fa6..995b006 100644 --- a/src/accessTokens.ts +++ b/src/accessTokens.ts @@ -1,5 +1,5 @@ -import type { DataToObject, S2RequestOptions } from "./common.js"; -import { S2Error } from "./error.js"; +import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; +import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type IssueAccessTokenData, @@ -9,6 +9,7 @@ import { type RevokeAccessTokenData, revokeAccessToken, } from "./generated/index.js"; +import { withRetries } from "./lib/retry.js"; export interface ListAccessTokensArgs extends DataToObject {} @@ -19,9 +20,11 @@ export interface RevokeAccessTokenArgs export class S2AccessTokens { readonly client: Client; + private readonly retryConfig?: RetryConfig; - constructor(client: Client) { + constructor(client: Client, retryConfig?: RetryConfig) { this.client = client; + this.retryConfig = retryConfig; } /** @@ -32,20 +35,17 @@ export class S2AccessTokens { * @param args.limit Max results (up to 1000) */ public async list(args?: ListAccessTokensArgs, options?: S2RequestOptions) { - const response = await listAccessTokens({ - client: this.client, - query: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + listAccessTokens({ + client: this.client, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -58,20 +58,17 @@ export class S2AccessTokens { * @param args.expires_at Expiration in ISO 8601; defaults to requestor's token expiry */ public async issue(args: IssueAccessTokenArgs, options?: S2RequestOptions) { - const response = await issueAccessToken({ - client: this.client, - body: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + issueAccessToken({ + client: this.client, + body: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -81,20 +78,17 @@ export class S2AccessTokens { * @param args.id Token ID to revoke */ public async revoke(args: RevokeAccessTokenArgs, options?: S2RequestOptions) { - const response = await revokeAccessToken({ - client: this.client, - path: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + revokeAccessToken({ + client: this.client, + path: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } } diff --git a/src/basin.ts b/src/basin.ts index e9ba382..bbc5c80 100644 --- a/src/basin.ts +++ b/src/basin.ts @@ -1,3 +1,4 @@ +import type { RetryConfig } from "./common.js"; import { createClient, createConfig } from "./generated/client/index.js"; import type { Client } from "./generated/client/types.gen.js"; import * as Redacted from "./lib/redacted.js"; @@ -8,6 +9,7 @@ import { S2Streams } from "./streams.js"; export class S2Basin { private readonly client: Client; private readonly transportConfig: TransportConfig; + private readonly retryConfig?: RetryConfig; public readonly name: string; public readonly streams: S2Streams; @@ -19,6 +21,7 @@ export class S2Basin { * @param accessToken Redacted access token from the parent `S2` client * @param baseUrl Base URL for the basin (e.g. `https://my-basin.b.aws.s2.dev/v1`) * @param includeBasinHeader Include the `S2-Basin` header with the request + * @param retryConfig Retry configuration inherited from parent S2 client */ constructor( name: string, @@ -26,12 +29,15 @@ export class S2Basin { accessToken: Redacted.Redacted; baseUrl: string; includeBasinHeader: boolean; + retryConfig?: RetryConfig; }, ) { this.name = name; + this.retryConfig = options.retryConfig; this.transportConfig = { baseUrl: options.baseUrl, accessToken: options.accessToken, + basinName: options.includeBasinHeader ? name : undefined, }; this.client = createClient( createConfig({ @@ -40,7 +46,8 @@ export class S2Basin { headers: options.includeBasinHeader ? { "s2-basin": name } : {}, }), ); - this.streams = new S2Streams(this.client); + + this.streams = new S2Streams(this.client, this.retryConfig); } /** @@ -48,10 +55,15 @@ export class S2Basin { * @param name Stream name */ public stream(name: string, options?: StreamOptions) { - return new S2Stream(name, this.client, { - ...this.transportConfig, - forceTransport: options?.forceTransport, - }); + return new S2Stream( + name, + this.client, + { + ...this.transportConfig, + forceTransport: options?.forceTransport, + }, + this.retryConfig, + ); } } diff --git a/src/basins.ts b/src/basins.ts index b1d494f..5eba10d 100644 --- a/src/basins.ts +++ b/src/basins.ts @@ -1,18 +1,23 @@ -import type { DataToObject, S2RequestOptions } from "./common.js"; -import { S2Error } from "./error.js"; +import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; +import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { + type BasinConfig, type CreateBasinData, + type CreateBasinResponse, createBasin, type DeleteBasinData, deleteBasin, type GetBasinConfigData, getBasinConfig, type ListBasinsData, + type ListBasinsResponse, listBasins, type ReconfigureBasinData, + type ReconfigureBasinResponse, reconfigureBasin, } from "./generated/index.js"; +import { withRetries } from "./lib/retry.js"; export interface ListBasinsArgs extends DataToObject {} export interface CreateBasinArgs extends DataToObject {} @@ -23,9 +28,11 @@ export interface ReconfigureBasinArgs export class S2Basins { private readonly client: Client; + private readonly retryConfig: RetryConfig; - constructor(client: Client) { + constructor(client: Client, retryConfig: RetryConfig) { this.client = client; + this.retryConfig = retryConfig; } /** @@ -35,21 +42,21 @@ export class S2Basins { * @param args.start_after Name to start after (for pagination) * @param args.limit Max results (up to 1000) */ - public async list(args?: ListBasinsArgs, options?: S2RequestOptions) { - const response = await listBasins({ - client: this.client, - query: args, - ...options, + public async list( + args?: ListBasinsArgs, + options?: S2RequestOptions, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + listBasins({ + client: this.client, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -60,21 +67,21 @@ export class S2Basins { * @param args.config Optional basin configuration (e.g. default stream config) * @param args.scope Basin scope */ - public async create(args: CreateBasinArgs, options?: S2RequestOptions) { - const response = await createBasin({ - client: this.client, - body: args, - ...options, + public async create( + args: CreateBasinArgs, + options?: S2RequestOptions, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + createBasin({ + client: this.client, + body: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -83,21 +90,21 @@ export class S2Basins { * * @param args.basin Basin name */ - public async getConfig(args: GetBasinConfigArgs, options?: S2RequestOptions) { - const response = await getBasinConfig({ - client: this.client, - path: args, - ...options, + public async getConfig( + args: GetBasinConfigArgs, + options?: S2RequestOptions, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + getBasinConfig({ + client: this.client, + path: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -106,22 +113,20 @@ export class S2Basins { * * @param args.basin Basin name */ - public async delete(args: DeleteBasinArgs, options?: S2RequestOptions) { - const response = await deleteBasin({ - client: this.client, - path: args, - ...options, + public async delete( + args: DeleteBasinArgs, + options?: S2RequestOptions, + ): Promise { + await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + deleteBasin({ + client: this.client, + path: args, + ...options, + throwOnError: true, + }), + ); }); - - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - - return response.data; } /** @@ -133,22 +138,19 @@ export class S2Basins { public async reconfigure( args: ReconfigureBasinArgs, options?: S2RequestOptions, - ) { - const response = await reconfigureBasin({ - client: this.client, - path: args, - body: args, - ...options, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + reconfigureBasin({ + client: this.client, + path: args, + body: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } } diff --git a/src/common.ts b/src/common.ts index decbbee..cd34f76 100644 --- a/src/common.ts +++ b/src/common.ts @@ -1,3 +1,36 @@ +/** + * Policy for retrying append operations. + * + * - `all`: Retry all append operations, including those that may have side effects + * - `noSideEffects`: Only retry append operations that are guaranteed to have no side effects + */ +export type AppendRetryPolicy = "all" | "noSideEffects"; + +/** + * Retry configuration for handling transient failures. + */ +export type RetryConfig = { + /** + * Maximum number of retry attempts. + * Set to 0 to disable retries. + * @default 3 + */ + maxAttempts?: number; + + /** + * Base delay in milliseconds between retry attempts. + * Uses exponential backoff: delay = retryBackoffDurationMs * (2 ^ attempt) + * @default 100 + */ + retryBackoffDurationMs?: number; + + /** + * Policy for retrying append operations. + * @default "noSideEffects" + */ + appendRetryPolicy?: AppendRetryPolicy; +}; + /** * Configuration for constructing the top-level `S2` client. * @@ -19,6 +52,12 @@ export type S2ClientOptions = { * Defaults to `https://{basin}.b.aws.s2.dev`. */ makeBasinBaseUrl?: (basin: string) => string; + /** + * Retry configuration for handling transient failures. + * Applies to management operations (basins, streams, tokens) and stream operations (read, append). + * @default { maxAttempts: 3, retryBackoffDurationMs: 100 } + */ + retry?: RetryConfig; }; /** diff --git a/src/error.ts b/src/error.ts index aacfcfd..50ca2d6 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,3 +1,71 @@ +function isConnectionError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + + if (error.message.includes("fetch failed")) { + return true; + } + + const cause = (error as any).cause; + if (cause && typeof cause === "object") { + const code = cause.code; + + // Common connection error codes from Node.js net module + const connectionErrorCodes = [ + "ECONNREFUSED", // Connection refused + "ENOTFOUND", // DNS lookup failed + "ETIMEDOUT", // Connection timeout + "ENETUNREACH", // Network unreachable + "EHOSTUNREACH", // Host unreachable + "ECONNRESET", // Connection reset by peer + "EPIPE", // Broken pipe + ]; + + if (connectionErrorCodes.includes(code)) { + return true; + } + } + + return false; +} + +export async function withS2Error(fn: () => Promise): Promise { + try { + return await fn(); + } catch (error) { + // Already S2Error? Rethrow + if (error instanceof S2Error) { + throw error; + } + + // Connection error? + if (isConnectionError(error)) { + const cause = (error as any).cause; + const code = cause?.code || "NETWORK_ERROR"; + throw new S2Error({ + message: `Connection failed: ${code}`, + // Could add a specific status or property for connection errors + status: 500, // or 0, or a constant + }); + } + + // Abort error? + if (error instanceof Error && error.name === "AbortError") { + throw new S2Error({ + message: "Request cancelled", + status: undefined, + }); + } + + // Other unknown errors + throw new S2Error({ + message: error instanceof Error ? error.message : "Unknown error", + status: 0, + }); + } +} + /** * Rich error type used by the SDK to surface HTTP and protocol errors. * diff --git a/src/generated/client/types.gen.ts b/src/generated/client/types.gen.ts index d68ab68..c037719 100644 --- a/src/generated/client/types.gen.ts +++ b/src/generated/client/types.gen.ts @@ -10,6 +10,7 @@ import type { Config as CoreConfig, } from '../core/types.gen.js'; import type { Middleware } from './utils.gen.js'; +import type {S2Error} from "../../error.js"; export type ResponseStyle = 'data' | 'fields'; @@ -204,7 +205,7 @@ export type Client = CoreClient< BuildUrlFn, SseFn > & { - interceptors: Middleware; + interceptors: Middleware; }; /** diff --git a/src/index.ts b/src/index.ts index a9905a0..ec82e31 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,12 @@ export type { } from "./basins.js"; export type { BatchOutput, BatchTransformArgs } from "./batch-transform.js"; export { BatchTransform } from "./batch-transform.js"; +export type { + AppendRetryPolicy, + RetryConfig, + S2ClientOptions, + S2RequestOptions, +} from "./common.js"; export { FencingTokenMismatchError, RangeNotSatisfiableError, diff --git a/src/lib/retry.ts b/src/lib/retry.ts new file mode 100644 index 0000000..5214424 --- /dev/null +++ b/src/lib/retry.ts @@ -0,0 +1,113 @@ +import createDebug from "debug"; +import type { RetryConfig } from "../common.js"; +import { S2Error } from "../error.js"; + +const debug = createDebug("s2:retry"); + +/** + * Default retry configuration. + */ +export const DEFAULT_RETRY_CONFIG: Required = { + maxAttempts: 3, + retryBackoffDurationMs: 100, + appendRetryPolicy: "noSideEffects", +}; + +/** + * Determines if an error should be retried based on its characteristics. + */ +export function isRetryable(error: S2Error): boolean { + // S2Error with retryable status code + const status = error.status; + if (status && (status >= 500 || status === 408)) { + return true; + } + return false; +} + +/** + * Calculates the delay before the next retry attempt using exponential backoff. + */ +export function calculateDelay(attempt: number, baseDelayMs: number): number { + // Exponential backoff: baseDelay * (2 ^ attempt) + const delay = baseDelayMs * Math.pow(2, attempt); + // Add jitter: random value between 0 and delay + const jitter = Math.random() * delay; + return Math.floor(delay + jitter); +} + +/** + * Sleeps for the specified duration. + */ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Executes an async function with automatic retry logic for transient failures. + * + * @param retryConfig Retry configuration (max attempts, backoff duration) + * @param fn The async function to execute + * @returns The result of the function + * @throws The last error if all retry attempts are exhausted + */ +export async function withRetries( + retryConfig: RetryConfig | undefined, + fn: () => Promise, + isPolicyCompliant: (config: RetryConfig, error: S2Error) => boolean = () => + true, +): Promise { + const config = { + ...DEFAULT_RETRY_CONFIG, + ...retryConfig, + }; + + // If maxAttempts is 0, don't retry at all + if (config.maxAttempts === 0) { + debug("maxAttempts is 0, retries disabled"); + return fn(); + } + + let lastError: S2Error | undefined = undefined; + + for (let attempt = 0; attempt <= config.maxAttempts; attempt++) { + try { + const result = await fn(); + if (attempt > 0) { + debug("succeeded after %d retries", attempt); + } + return result; + } catch (error) { + // withRetry only handles S2Errors (withS2Error should be called first) + if (!(error instanceof S2Error)) { + debug("non-S2Error thrown, rethrowing immediately: %s", error); + throw error; + } + + lastError = error; + + // Don't retry if this is the last attempt + if (attempt === config.maxAttempts) { + debug("max attempts exhausted, throwing error"); + break; + } + + // Check if error is retryable + if (!isPolicyCompliant(config, lastError) || !isRetryable(lastError)) { + debug("error not retryable, throwing immediately"); + throw error; + } + + // Calculate delay and wait before retrying + const delay = calculateDelay(attempt, config.retryBackoffDurationMs); + debug( + "retryable error, backing off for %dms, status=%s", + delay, + error.status, + ); + await sleep(delay); + } + } + + throw lastError; +} diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 3b90c18..e9ea473 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -38,6 +38,7 @@ export class FetchReadSession< args?: ReadArgs, options?: S2RequestOptions, ) { + console.log("FetchReadSession.create", name, args); const { as, ...queryParams } = args ?? {}; const response = await read({ client, @@ -45,6 +46,8 @@ export class FetchReadSession< stream: name, }, headers: { + // Note: These headers are merged with defaults via mergeHeaders(). + // Authorization is added afterward via setAuthParams(), so it's preserved. accept: "text/event-stream", ...(as === "bytes" ? { "s2-format": "base64" } : {}), }, @@ -490,6 +493,7 @@ export class FetchTransport implements SessionTransport { createConfig({ baseUrl: config.baseUrl, auth: () => Redacted.value(config.accessToken), + headers: config.basinName ? { "s2-basin": config.basinName } : {}, }), ); this.transportConfig = config; diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index c0507db..9858a91 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -83,6 +83,7 @@ export class S2STransport implements SessionTransport { createConfig({ baseUrl: config.baseUrl, auth: () => Redacted.value(config.accessToken), + headers: config.basinName ? { "s2-basin": config.basinName } : {}, }), ); this.transportConfig = config; @@ -98,6 +99,7 @@ export class S2STransport implements SessionTransport { this.transportConfig.accessToken, stream, () => this.getConnection(), + this.transportConfig.basinName, sessionOptions, requestOptions, ); @@ -115,6 +117,7 @@ export class S2STransport implements SessionTransport { args, options, () => this.getConnection(), + this.transportConfig.basinName, ); } @@ -195,6 +198,7 @@ class S2SReadSession args: ReadArgs | undefined, options: S2RequestOptions | undefined, getConnection: () => Promise, + basinName?: string, ): Promise> { const url = new URL(baseUrl); return new S2SReadSession( @@ -204,6 +208,7 @@ class S2SReadSession url, options, getConnection, + basinName, ); } @@ -214,6 +219,7 @@ class S2SReadSession private url: URL, private options: S2RequestOptions | undefined, private getConnection: () => Promise, + private basinName?: string, ) { // Initialize parser and textDecoder before super() call const parser = new S2SFrameParser(); @@ -224,6 +230,7 @@ class S2SReadSession super({ start: async (controller) => { let controllerClosed = false; + let responseCode: number | undefined; const safeClose = () => { if (!controllerClosed) { controllerClosed = true; @@ -275,6 +282,7 @@ class S2SReadSession authorization: `Bearer ${Redacted.value(authToken)}`, accept: "application/protobuf", "content-type": "s2s/proto", + ...(basinName ? { "s2-basin": basinName } : {}), }); http2Stream = stream; @@ -285,7 +293,31 @@ class S2SReadSession } }); + stream.on("response", (headers) => { + responseCode = headers[":status"] ?? 500; + }); + stream.on("data", (chunk: Buffer) => { + if ((responseCode ?? 500) >= 400) { + const errorText = textDecoder.decode(chunk); + try { + const errorJson = JSON.parse(errorText); + safeError( + new S2Error({ + message: errorJson.message ?? "Unknown error", + code: errorJson.code, + status: responseCode, + }), + ); + } catch { + safeError( + new S2Error({ + message: errorText || "Unknown error", + status: responseCode, + }), + ); + } + } // Buffer already extends Uint8Array in Node.js, no need to convert parser.push(chunk); @@ -540,6 +572,7 @@ class S2SAppendSession bearerToken: Redacted.Redacted, streamName: string, getConnection: () => Promise, + basinName: string | undefined, sessionOptions?: AppendSessionOptions, requestOptions?: S2RequestOptions, ): Promise { @@ -548,6 +581,7 @@ class S2SAppendSession bearerToken, streamName, getConnection, + basinName, sessionOptions, requestOptions, ); @@ -558,6 +592,7 @@ class S2SAppendSession private authToken: Redacted.Redacted, private streamName: string, private getConnection: () => Promise, + private basinName?: string, sessionOptions?: AppendSessionOptions, private options?: S2RequestOptions, ) { @@ -666,6 +701,7 @@ class S2SAppendSession authorization: `Bearer ${Redacted.value(this.authToken)}`, "content-type": "s2s/proto", accept: "application/protobuf", + ...(this.basinName ? { "s2-basin": this.basinName } : {}), }); this.http2Stream = stream; diff --git a/src/lib/stream/types.ts b/src/lib/stream/types.ts index bb700cc..5b4e9fb 100644 --- a/src/lib/stream/types.ts +++ b/src/lib/stream/types.ts @@ -105,4 +105,8 @@ export interface TransportConfig { baseUrl: string; accessToken: Redacted.Redacted; forceTransport?: SessionTransports; + /** + * Basin name to include in s2-basin header when using account endpoint + */ + basinName?: string; } diff --git a/src/metrics.ts b/src/metrics.ts index f381598..0c32153 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,5 +1,5 @@ -import type { DataToObject, S2RequestOptions } from "./common.js"; -import { S2Error } from "./error.js"; +import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; +import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AccountMetricsData, @@ -9,6 +9,7 @@ import { type StreamMetricsData, streamMetrics, } from "./generated/index.js"; +import { withRetries } from "./lib/retry.js"; export interface AccountMetricsArgs extends DataToObject {} export interface BasinMetricsArgs extends DataToObject {} @@ -16,9 +17,11 @@ export interface StreamMetricsArgs extends DataToObject {} export class S2Metrics { readonly client: Client; + private readonly retryConfig?: RetryConfig; - constructor(client: Client) { + constructor(client: Client, retryConfig?: RetryConfig) { this.client = client; + this.retryConfig = retryConfig; } /** @@ -30,20 +33,17 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async account(args: AccountMetricsArgs, options?: S2RequestOptions) { - const response = await accountMetrics({ - client: this.client, - query: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + accountMetrics({ + client: this.client, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -57,21 +57,18 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async basin(args: BasinMetricsArgs, options?: S2RequestOptions) { - const response = await basinMetrics({ - client: this.client, - path: args, - query: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + basinMetrics({ + client: this.client, + path: args, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -86,21 +83,18 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async stream(args: StreamMetricsArgs, options?: S2RequestOptions) { - const response = await streamMetrics({ - client: this.client, - path: args, - query: args, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + streamMetrics({ + client: this.client, + path: args, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } } diff --git a/src/s2.ts b/src/s2.ts index a446782..b436ec1 100644 --- a/src/s2.ts +++ b/src/s2.ts @@ -1,7 +1,8 @@ import { S2AccessTokens } from "./accessTokens.js"; import { S2Basin } from "./basin.js"; import { S2Basins } from "./basins.js"; -import type { S2ClientOptions } from "./common.js"; +import type { RetryConfig, S2ClientOptions } from "./common.js"; +import { S2Error } from "./error.js"; import { createClient, createConfig } from "./generated/client/index.js"; import type { Client } from "./generated/client/types.gen.js"; import * as Redacted from "./lib/redacted.js"; @@ -21,6 +22,7 @@ export class S2 { private readonly client: Client; private readonly makeBasinBaseUrl: (basin: string) => string; private readonly includeBasinHeader: boolean; + private readonly retryConfig: RetryConfig; /** * Account-scoped basin management operations. @@ -40,15 +42,26 @@ export class S2 { */ constructor(options: S2ClientOptions) { this.accessToken = Redacted.make(options.accessToken); + this.retryConfig = options.retry ?? {}; this.client = createClient( createConfig({ baseUrl: options.baseUrl ?? defaultBaseUrl, auth: () => Redacted.value(this.accessToken), + throwOnError: true, }), ); - this.basins = new S2Basins(this.client); - this.accessTokens = new S2AccessTokens(this.client); - this.metrics = new S2Metrics(this.client); + + this.client.interceptors.error.use((err, res, req, opt) => { + return new S2Error({ + message: err instanceof Error ? err.message : "Unknown error", + code: res.statusText, + status: res.status, + }); + }); + + this.basins = new S2Basins(this.client, this.retryConfig); + this.accessTokens = new S2AccessTokens(this.client, this.retryConfig); + this.metrics = new S2Metrics(this.client, this.retryConfig); this.makeBasinBaseUrl = options.makeBasinBaseUrl ?? defaultMakeBasinBaseUrl; this.includeBasinHeader = !!options.makeBasinBaseUrl; } @@ -63,6 +76,7 @@ export class S2 { accessToken: this.accessToken, baseUrl: this.makeBasinBaseUrl(name), includeBasinHeader: this.includeBasinHeader, + retryConfig: this.retryConfig, }); } } diff --git a/src/stream.ts b/src/stream.ts index aca6e71..ea2630d 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,7 +1,8 @@ -import type { S2RequestOptions } from "./common.js"; -import { S2Error } from "./error.js"; +import type { RetryConfig, S2RequestOptions } from "./common.js"; +import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AppendAck, checkTail } from "./generated/index.js"; +import { withRetries } from "./lib/retry.js"; import { createSessionTransport } from "./lib/stream/factory.js"; import { streamAppend, @@ -18,18 +19,28 @@ import type { SessionTransport, TransportConfig, } from "./lib/stream/types.js"; +import createDebug from "debug"; + +const debug = createDebug("s2:stream"); export class S2Stream { private readonly client: Client; private readonly transportConfig: TransportConfig; + private readonly retryConfig?: RetryConfig; private _transport?: SessionTransport; public readonly name: string; - constructor(name: string, client: Client, transportConfig: TransportConfig) { + constructor( + name: string, + client: Client, + transportConfig: TransportConfig, + retryConfig?: RetryConfig, + ) { this.name = name; this.client = client; this.transportConfig = transportConfig; + this.retryConfig = retryConfig; } /** @@ -48,22 +59,19 @@ export class S2Stream { * Returns the next sequence number and timestamp to be assigned (`tail`). */ public async checkTail(options?: S2RequestOptions) { - const response = await checkTail({ - client: this.client, - path: { - stream: this.name, - }, - ...options, + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + checkTail({ + client: this.client, + path: { + stream: this.name, + }, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -79,7 +87,9 @@ export class S2Stream { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - return await streamRead(this.name, this.client, args, options); + return await withRetries(this.retryConfig, async () => { + return await streamRead(this.name, this.client, args, options); + }); } /** * Append one or more records to the stream. @@ -100,7 +110,25 @@ export class S2Stream { args?: Omit, options?: S2RequestOptions, ): Promise { - return await streamAppend(this.name, this.client, records, args, options); + return await withRetries( + this.retryConfig, + async () => { + return await streamAppend( + this.name, + this.client, + records, + args, + options, + ); + }, + (config, error) => { + if ((config.appendRetryPolicy ?? "noSideEffects") === "noSideEffects") { + return !!args?.match_seq_num; + } else { + return true; + } + }, + ); } /** * Open a streaming read session diff --git a/src/streams.ts b/src/streams.ts index 78b985e..0167388 100644 --- a/src/streams.ts +++ b/src/streams.ts @@ -1,18 +1,23 @@ -import type { DataToObject, S2RequestOptions } from "./common.js"; -import { S2Error } from "./error.js"; +import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; +import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type CreateStreamData, + type CreateStreamResponse, createStream, type DeleteStreamData, deleteStream, type GetStreamConfigData, getStreamConfig, type ListStreamsData, + type ListStreamsResponse, listStreams, type ReconfigureStreamData, + type ReconfigureStreamResponse, reconfigureStream, + type StreamConfig, } from "./generated/index.js"; +import { withRetries } from "./lib/retry.js"; export interface ListStreamsArgs extends DataToObject {} export interface CreateStreamArgs extends DataToObject {} @@ -24,8 +29,11 @@ export interface ReconfigureStreamArgs export class S2Streams { private readonly client: Client; - constructor(client: Client) { + private readonly retryConfig?: RetryConfig; + + constructor(client: Client, retryConfig?: RetryConfig) { this.client = client; + this.retryConfig = retryConfig; } /** @@ -35,21 +43,21 @@ export class S2Streams { * @param args.start_after Name to start after (for pagination) * @param args.limit Max results (up to 1000) */ - public async list(args?: ListStreamsArgs, options?: S2RequestOptions) { - const response = await listStreams({ - client: this.client, - query: args, - ...options, + public async list( + args?: ListStreamsArgs, + options?: S2RequestOptions, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + listStreams({ + client: this.client, + query: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -59,21 +67,21 @@ export class S2Streams { * @param args.stream Stream name (1-512 bytes, unique within the basin) * @param args.config Stream configuration (retention, storage class, timestamping, delete-on-empty) */ - public async create(args: CreateStreamArgs, options?: S2RequestOptions) { - const response = await createStream({ - client: this.client, - body: args, - ...options, + public async create( + args: CreateStreamArgs, + options?: S2RequestOptions, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + createStream({ + client: this.client, + body: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -85,21 +93,18 @@ export class S2Streams { public async getConfig( args: GetStreamConfigArgs, options?: S2RequestOptions, - ) { - const response = await getStreamConfig({ - client: this.client, - path: args, - ...options, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + getStreamConfig({ + client: this.client, + path: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } @@ -108,22 +113,20 @@ export class S2Streams { * * @param args.stream Stream name */ - public async delete(args: DeleteStreamArgs, options?: S2RequestOptions) { - const response = await deleteStream({ - client: this.client, - path: args, - ...options, + public async delete( + args: DeleteStreamArgs, + options?: S2RequestOptions, + ): Promise { + await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + deleteStream({ + client: this.client, + path: args, + ...options, + throwOnError: true, + }), + ); }); - - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - - return response.data; } /** @@ -135,22 +138,19 @@ export class S2Streams { public async reconfigure( args: ReconfigureStreamArgs, options?: S2RequestOptions, - ) { - const response = await reconfigureStream({ - client: this.client, - path: args, - body: args, - ...options, + ): Promise { + const response = await withRetries(this.retryConfig, async () => { + return await withS2Error(async () => + reconfigureStream({ + client: this.client, + path: args, + body: args, + ...options, + throwOnError: true, + }), + ); }); - if (response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } - return response.data; } } diff --git a/src/tests/appendSession.e2e.test.ts b/src/tests/appendSession.e2e.test.ts index 61f141a..ed3bd13 100644 --- a/src/tests/appendSession.e2e.test.ts +++ b/src/tests/appendSession.e2e.test.ts @@ -11,23 +11,17 @@ describe("AppendSession Integration Tests", () => { beforeAll(() => { const token = process.env.S2_ACCESS_TOKEN; - if (!token) { + const basin = process.env.S2_BASIN; + if (!token || !basin) { throw new Error( - "S2_ACCESS_TOKEN environment variable is required for integration tests", + "S2_ACCESS_TOKEN and S2_BASIN environment variables are required for e2e tests", ); } s2 = new S2({ accessToken: token }); + basinName = basin; }); beforeAll(async () => { - // Get or use an existing basin - const basins = await s2.basins.list(); - if (!basins.basins || basins.basins.length === 0) { - throw new Error("No basins found. Please create a basin first."); - } - basinName = basins.basins[0]!.name; - expect(basinName).toBeTruthy(); - // Use a unique stream name for each test run const timestamp = Date.now(); streamName = `integration-test-append-${timestamp}`; diff --git a/src/tests/retry.test.ts b/src/tests/retry.test.ts new file mode 100644 index 0000000..cfbf17f --- /dev/null +++ b/src/tests/retry.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from "vitest"; +import { S2Error } from "../error.js"; +import { DEFAULT_RETRY_CONFIG, withRetries } from "../lib/retry.js"; + +describe("Retry Logic", () => { + describe("withRetry", () => { + it("should succeed on first attempt", async () => { + const fn = vi.fn().mockResolvedValue("success"); + const result = await withRetries(undefined, fn); + + expect(result).toBe("success"); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("should retry on S2Error with 5xx status", async () => { + const fn = vi + .fn() + .mockRejectedValueOnce( + new S2Error({ message: "Server error", status: 503 }), + ) + .mockResolvedValue("success"); + + const result = await withRetries( + { maxAttempts: 3, retryBackoffDurationMs: 1 }, + fn, + ); + + expect(result).toBe("success"); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it("should not retry on S2Error with 4xx status", async () => { + const error = new S2Error({ message: "Bad request", status: 400 }); + const fn = vi.fn().mockRejectedValue(error); + + await expect( + withRetries({ maxAttempts: 3, retryBackoffDurationMs: 1 }, fn), + ).rejects.toThrow(error); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("should retry on 408 Request Timeout", async () => { + const fn = vi + .fn() + .mockRejectedValueOnce( + new S2Error({ message: "Request timeout", status: 408 }), + ) + .mockResolvedValue("success"); + + const result = await withRetries( + { maxAttempts: 3, retryBackoffDurationMs: 1 }, + fn, + ); + + expect(result).toBe("success"); + expect(fn).toHaveBeenCalledTimes(2); + }); + + // Note: Tests for connection and abort errors will be added after implementing + // error kind discrimination (see claude.md). For now, withS2Error converts these + // to S2Error with status=undefined, and isRetryable doesn't handle them yet. + + it("should exhaust retries and throw last error", async () => { + const error = new S2Error({ message: "Server error", status: 503 }); + const fn = vi.fn().mockRejectedValue(error); + + await expect( + withRetries({ maxAttempts: 2, retryBackoffDurationMs: 1 }, fn), + ).rejects.toThrow(error); + + // Initial attempt + 2 retries = 3 calls + expect(fn).toHaveBeenCalledTimes(3); + }); + + it("should not retry when maxAttempts is 0", async () => { + const error = new S2Error({ message: "Server error", status: 503 }); + const fn = vi.fn().mockRejectedValue(error); + + await expect( + withRetries({ maxAttempts: 0, retryBackoffDurationMs: 1 }, fn), + ).rejects.toThrow(error); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("should use default config when not provided", async () => { + const fn = vi.fn().mockResolvedValue("success"); + const result = await withRetries(undefined, fn); + + expect(result).toBe("success"); + expect(fn).toHaveBeenCalledTimes(1); + }); + }); + + describe("DEFAULT_RETRY_CONFIG", () => { + it("should have correct default values", () => { + expect(DEFAULT_RETRY_CONFIG.maxAttempts).toBe(3); + expect(DEFAULT_RETRY_CONFIG.retryBackoffDurationMs).toBe(100); + expect(DEFAULT_RETRY_CONFIG.appendRetryPolicy).toBe("noSideEffects"); + }); + }); +}); From a1aebf09f9dccd7ffe8bbff9e47dd77b34762475 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 4 Nov 2025 08:34:21 -0800 Subject: [PATCH 02/26] a --- src/lib/stream/transport/fetch/index.ts | 2 -- src/lib/stream/transport/s2s/index.ts | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index e9ea473..1760b08 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -46,8 +46,6 @@ export class FetchReadSession< stream: name, }, headers: { - // Note: These headers are merged with defaults via mergeHeaders(). - // Authorization is added afterward via setAuthParams(), so it's preserved. accept: "text/event-stream", ...(as === "bytes" ? { "s2-format": "base64" } : {}), }, diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 9858a91..4214a06 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -294,6 +294,8 @@ class S2SReadSession }); stream.on("response", (headers) => { + // Cache the status. + // This informs whether we should attempt to parse s2s frames in the "data" handler. responseCode = headers[":status"] ?? 500; }); From 63841a3e60f651463a3d6ceec686c40df8b3d1cc Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 4 Nov 2025 15:06:04 -0800 Subject: [PATCH 03/26] a --- package-lock.json | 4 +++- package.json | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index e02494d..d035618 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.17.0", "license": "Apache-2.0", "dependencies": { + "@protobuf-ts/runtime": "^2.11.1", "debug": "^4.4.3" }, "devDependencies": { @@ -695,7 +696,8 @@ }, "node_modules/@protobuf-ts/runtime": { "version": "2.11.1", - "dev": true, + "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz", + "integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@protobuf-ts/runtime-rpc": { diff --git a/package.json b/package.json index a18bfe9..af4f9d1 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "typescript": "^5.9.3" }, "dependencies": { + "@protobuf-ts/runtime": "^2.11.1", "debug": "^4.4.3" } } From 670e4927be8932c4f71ea944086206b3ee5817ff Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 21:29:42 -0800 Subject: [PATCH 04/26] retry --- examples/image.ts | 16 +- examples/read.ts | 2 +- package.json | 8 +- src/basin.ts | 1 + src/batch-transform.ts | 4 + src/error.ts | 95 +- src/lib/result.ts | 58 + src/lib/retry.ts | 1025 ++++++++++++++++- src/lib/retry.ts.old | 1408 +++++++++++++++++++++++ src/lib/stream/transport/fetch/index.ts | 522 +++++---- src/lib/stream/transport/s2s/framing.ts | 2 +- src/lib/stream/transport/s2s/index.ts | 797 ++++++------- src/lib/stream/types.ts | 70 +- src/stream.ts | 8 +- src/tests/appendSession.test.ts | 31 +- src/tests/batcher-session.test.ts | 41 +- src/tests/readSession.e2e.test.ts | 10 +- src/tests/retryAppendSession.test.ts | 367 ++++++ src/tests/retryReadSession.test.ts | 445 +++++++ src/utils.ts | 5 +- 20 files changed, 4162 insertions(+), 753 deletions(-) create mode 100644 src/lib/result.ts create mode 100644 src/lib/retry.ts.old create mode 100644 src/tests/retryAppendSession.test.ts create mode 100644 src/tests/retryReadSession.test.ts diff --git a/examples/image.ts b/examples/image.ts index ea416d0..d2aef15 100644 --- a/examples/image.ts +++ b/examples/image.ts @@ -29,8 +29,20 @@ function rechunkStream( }); } +// const s2 = new S2({ +// accessToken: process.env.S2_ACCESS_TOKEN!, +// }); + const s2 = new S2({ accessToken: process.env.S2_ACCESS_TOKEN!, + baseUrl: `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, + makeBasinBaseUrl: (basinName) => + `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, + retry: { + maxAttempts: 10, + retryBackoffDurationMs: 1000, + appendRetryPolicy: "all", + }, }); const basinName = process.env.S2_BASIN; @@ -66,8 +78,8 @@ let append = await image // Collect records into batches. .pipeThrough( new BatchTransform({ - lingerDurationMillis: 50, - match_seq_num: startAt.tail.seq_num, + lingerDurationMillis: 1, + match_seq_num: 0, }), ) // Write to the S2 stream. diff --git a/examples/read.ts b/examples/read.ts index 047940e..9ab3e10 100644 --- a/examples/read.ts +++ b/examples/read.ts @@ -24,7 +24,7 @@ if (streams.streams[0]) { for await (const record of readSession) { console.log(`[seq ${record.seq_num}] ${record.body}`); - console.log("new tail", readSession.lastReadPosition()?.seq_num); + console.log("new tail", readSession.nextReadPosition()?.seq_num); } console.log("Done reading"); } diff --git a/package.json b/package.json index af4f9d1..9790c5d 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "LICENSE" ], "dependencies": { - "@protobuf-ts/runtime": "^2.11.1" + "@protobuf-ts/runtime": "^2.11.1", + "debug": "^4.4.3" }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.2", @@ -63,8 +64,5 @@ "peerDependencies": { "typescript": "^5.9.3" }, - "dependencies": { - "@protobuf-ts/runtime": "^2.11.1", - "debug": "^4.4.3" - } + "packageManager": "pnpm@10.12.4+sha512.5ea8b0deed94ed68691c9bad4c955492705c5eeb8a87ef86bc62c74a26b037b08ff9570f108b2e4dbd1dd1a9186fea925e527f141c648e85af45631074680184" } diff --git a/src/basin.ts b/src/basin.ts index bbc5c80..c3e7c43 100644 --- a/src/basin.ts +++ b/src/basin.ts @@ -38,6 +38,7 @@ export class S2Basin { baseUrl: options.baseUrl, accessToken: options.accessToken, basinName: options.includeBasinHeader ? name : undefined, + retry: options.retryConfig, }; this.client = createClient( createConfig({ diff --git a/src/batch-transform.ts b/src/batch-transform.ts index 285f56f..aab7a96 100644 --- a/src/batch-transform.ts +++ b/src/batch-transform.ts @@ -1,6 +1,9 @@ +import createDebug from "debug"; import { S2Error } from "./error.js"; import { AppendRecord, meteredSizeBytes } from "./utils.js"; +const debug = createDebug("s2:batch_transform"); + export interface BatchTransformArgs { /** Duration in milliseconds to wait before flushing a batch (default: 5ms) */ lingerDurationMillis?: number; @@ -157,6 +160,7 @@ export class BatchTransform extends TransformStream { if (match_seq_num !== undefined) { batch.match_seq_num = match_seq_num; } + debug!({ batch }); this.controller.enqueue(batch); } diff --git a/src/error.ts b/src/error.ts index 50ca2d6..44e2ca5 100644 --- a/src/error.ts +++ b/src/error.ts @@ -8,61 +8,66 @@ function isConnectionError(error: unknown): boolean { } const cause = (error as any).cause; + let code = (error as any).code; + // TODO check if code exists if (cause && typeof cause === "object") { - const code = cause.code; - - // Common connection error codes from Node.js net module - const connectionErrorCodes = [ - "ECONNREFUSED", // Connection refused - "ENOTFOUND", // DNS lookup failed - "ETIMEDOUT", // Connection timeout - "ENETUNREACH", // Network unreachable - "EHOSTUNREACH", // Host unreachable - "ECONNRESET", // Connection reset by peer - "EPIPE", // Broken pipe - ]; - - if (connectionErrorCodes.includes(code)) { - return true; - } + code = cause.code; + } + + // Common connection error codes from Node.js net module + const connectionErrorCodes = [ + "ECONNREFUSED", // Connection refused + "ENOTFOUND", // DNS lookup failed + "ETIMEDOUT", // Connection timeout + "ENETUNREACH", // Network unreachable + "EHOSTUNREACH", // Host unreachable + "ECONNRESET", // Connection reset by peer + "EPIPE", // Broken pipe + ]; + + if (connectionErrorCodes.includes(code)) { + return true; } return false; } +export function s2Error(error: any): S2Error { + if (error instanceof S2Error) { + return error; + } + + // Connection error? + if (isConnectionError(error)) { + const cause = (error as any).cause; + const code = cause?.code || "NETWORK_ERROR"; + return new S2Error({ + message: `Connection failed: ${code}`, + // Could add a specific status or property for connection errors + status: 500, // or 0, or a constant + }); + } + + // Abort error? + if (error instanceof Error && error.name === "AbortError") { + return new S2Error({ + message: "Request cancelled", + status: undefined, + }); + } + + // Other unknown errors + return new S2Error({ + message: error instanceof Error ? error.message : "Unknown error", + status: 0, + }); +} + export async function withS2Error(fn: () => Promise): Promise { try { return await fn(); } catch (error) { - // Already S2Error? Rethrow - if (error instanceof S2Error) { - throw error; - } - - // Connection error? - if (isConnectionError(error)) { - const cause = (error as any).cause; - const code = cause?.code || "NETWORK_ERROR"; - throw new S2Error({ - message: `Connection failed: ${code}`, - // Could add a specific status or property for connection errors - status: 500, // or 0, or a constant - }); - } - - // Abort error? - if (error instanceof Error && error.name === "AbortError") { - throw new S2Error({ - message: "Request cancelled", - status: undefined, - }); - } - - // Other unknown errors - throw new S2Error({ - message: error instanceof Error ? error.message : "Unknown error", - status: 0, - }); + throw s2Error(error); } } diff --git a/src/lib/result.ts b/src/lib/result.ts new file mode 100644 index 0000000..0844375 --- /dev/null +++ b/src/lib/result.ts @@ -0,0 +1,58 @@ +/** + * Result types for AppendSession operations. + * Using discriminated unions for ergonomic error handling with TypeScript control flow analysis. + */ + +import { S2Error } from "../error.js"; +import type { AppendAck } from "../generated/index.js"; + +/** + * Result of an append operation. + * Use discriminated union pattern: check `result.ok` to access either `value` or `error`. + */ +export type AppendResult = + | { ok: true; value: AppendAck } + | { ok: false; error: S2Error }; + +/** + * Result of a close operation. + */ +export type CloseResult = { ok: true } | { ok: false; error: S2Error }; + +/** + * Constructs a successful append result. + */ +export function ok(value: AppendAck): AppendResult { + return { ok: true, value }; +} + +/** + * Constructs a failed append result. + */ +export function err(error: S2Error): AppendResult { + return { ok: false, error }; +} + +/** + * Constructs a successful close result. + */ +export function okClose(): CloseResult { + return { ok: true }; +} + +/** + * Constructs a failed close result. + */ +export function errClose(error: S2Error): CloseResult { + return { ok: false, error }; +} + +/** + * Type guard to check if a result is successful. + * Mainly for internal use; prefer `result.ok` for public API. + */ +export function isOk( + result: { ok: true; value: T } | { ok: false; error: S2Error }, +): result is { ok: true; value: T } { + return result.ok; +} diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 5214424..a4086e5 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -1,27 +1,62 @@ import createDebug from "debug"; import type { RetryConfig } from "../common.js"; -import { S2Error } from "../error.js"; +import { S2Error, s2Error, withS2Error } from "../error.js"; +import type { AppendAck, StreamPosition } from "../generated/index.js"; +import { meteredSizeBytes } from "../utils.js"; +import type { AppendResult, CloseResult } from "./result.js"; +import { err, errClose, ok, okClose } from "./result.js"; +import type { + AcksStream, + AppendArgs, + AppendRecord, + AppendSession, + AppendSessionOptions, + ReadArgs, + ReadRecord, + ReadSession, + TransportAppendSession, + TransportReadSession, +} from "./stream/types.js"; const debug = createDebug("s2:retry"); /** * Default retry configuration. */ -export const DEFAULT_RETRY_CONFIG: Required = { +export const DEFAULT_RETRY_CONFIG: Required & { + requestTimeoutMs: number; +} = { maxAttempts: 3, retryBackoffDurationMs: 100, appendRetryPolicy: "noSideEffects", + requestTimeoutMs: 5000, // 5 seconds }; +const RETRYABLE_STATUS_CODES = new Set([ + 408, // request_timeout + 429, // too_many_requests + 500, // internal_server_error + 502, // bad_gateway + 503, // service_unavailable +]); + /** * Determines if an error should be retried based on its characteristics. + * 400-level errors (except 408, 429) are non-retryable validation/client errors. */ export function isRetryable(error: S2Error): boolean { - // S2Error with retryable status code - const status = error.status; - if (status && (status >= 500 || status === 408)) { + if (!error.status) return false; + + // Explicit retryable codes (including some 4xx like 408, 429) + if (RETRYABLE_STATUS_CODES.has(error.status)) { return true; } + + // 400-level errors are generally non-retryable (validation, bad request) + if (error.status >= 400 && error.status < 500) { + return false; + } + return false; } @@ -111,3 +146,983 @@ export async function withRetries( throw lastError; } +export class RetryReadSession + extends ReadableStream> + implements ReadSession +{ + private _nextReadPosition: StreamPosition | undefined = undefined; + + private _recordsRead: number = 0; + private _bytesRead: number = 0; + + static async create( + generator: ( + args: ReadArgs, + ) => Promise>, + args: ReadArgs = {}, + config?: RetryConfig, + ) { + return new RetryReadSession(args, generator, config); + } + + private constructor( + args: ReadArgs, + generator: ( + args: ReadArgs, + ) => Promise>, + config?: RetryConfig, + ) { + const retryConfig = { + ...DEFAULT_RETRY_CONFIG, + ...config, + }; + let session: TransportReadSession | undefined = undefined; + const startTimeMs = performance.now(); // Capture start time before super() + super({ + start: async (controller) => { + let nextArgs = { ...args }; + let attempt = 0; + + while (true) { + session = await generator(nextArgs); + const reader = session.getReader(); + + while (true) { + const { done, value: result } = await reader.read(); + if (done) { + reader.releaseLock(); + controller.close(); + return; + } + + // Check if result is an error + if (!result.ok) { + reader.releaseLock(); + const error = result.error; + + // Check if we can retry (track session attempts, not record reads) + if (isRetryable(error) && attempt < retryConfig.maxAttempts) { + if (this._nextReadPosition) { + nextArgs.seq_num = this._nextReadPosition.seq_num; + } + if (nextArgs.count) { + nextArgs.count = + this._recordsRead === undefined + ? nextArgs.count + : nextArgs.count - this._recordsRead; + } + if (nextArgs.bytes) { + nextArgs.bytes = + this._bytesRead === undefined + ? nextArgs.bytes + : nextArgs.bytes - this._bytesRead; + } + // Adjust wait to account for elapsed time. + // If user specified wait=10s and we've already spent 5s (including backoff), + // we should only wait another 5s on retry to honor the original time budget. + if (nextArgs.wait !== undefined) { + const elapsedSeconds = + (performance.now() - startTimeMs) / 1000; + nextArgs.wait = Math.max(0, nextArgs.wait - elapsedSeconds); + } + const delay = calculateDelay( + attempt, + retryConfig.retryBackoffDurationMs, + ); + debug("will retry after %dms, status=%s", delay, error.status); + await sleep(delay); + attempt++; + break; // Break inner loop to retry + } + + // Error is not retryable or attempts exhausted + debug("error in retry loop: %s", error); + controller.error(error); + return; + } + + // Success: enqueue the record (don't reset attempt counter - track session attempts) + const record = result.value; + this._nextReadPosition = { + seq_num: record.seq_num + 1, + timestamp: record.timestamp, + }; + this._recordsRead++; + this._bytesRead += meteredSizeBytes(record); + attempt = 0; + + controller.enqueue(record); + } + } + }, + cancel: async () => { + session?.cancel(); + }, + }); + } + + async [Symbol.asyncDispose]() { + await this.cancel("disposed"); + } + + // Polyfill for older browsers / Node.js environments + [Symbol.asyncIterator](): AsyncIterableIterator> { + const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; + if (typeof fn === "function") return fn.call(this); + const reader = this.getReader(); + return { + next: async () => { + const r = await reader.read(); + if (r.done) { + reader.releaseLock(); + return { done: true, value: undefined }; + } + return { done: false, value: r.value }; + }, + throw: async (e) => { + await reader.cancel(e); + reader.releaseLock(); + return { done: true, value: undefined }; + }, + return: async () => { + await reader.cancel("done"); + reader.releaseLock(); + return { done: true, value: undefined }; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + } + + lastObservedTail(): StreamPosition | undefined { + return undefined; + } + + nextReadPosition(): StreamPosition | undefined { + return undefined; + } +} + +/** + * RetryAppendSession wraps an underlying AppendSession with automatic retry logic. + * + * Architecture: + * - All writes (submit() and writable.write()) are serialized through inflightQueue + * - inflightQueue tracks batches that have been submitted but not yet acked + * - Background ack reader consumes acks and matches them FIFO with inflightQueue + * - On error, _initSession() recreates session and re-transmits all inflightQueue batches + * - Ack timeout is fatal: if no ack arrives within the timeout window, + * the session aborts and rejects queued writers + * + * Flow for a successful append: + * 1. submit(records) adds batch to inflightQueue with promise resolvers + * 2. Calls underlying session.submit() to send batch + * 3. Background reader receives ack, validates record count + * 4. Resolves promise, removes from inflightQueue, forwards ack to user + * + * Flow for a failed append: + * 1. submit(records) adds batch to inflightQueue + * 2. Calls underlying session.submit() which fails + * 3. Checks if retryable (status code, retry policy, idempotency) + * 4. Calls _initSession() which closes old session, creates new session + * 5. _initSession() re-transmits ALL batches in inflightQueue (recovery) + * 6. Background reader receives acks for recovered batches + * 7. Original submit() call's promise is resolved by background reader + * + * Invariants: + * - Exactly one ack per batch in FIFO order + * - Ack record count matches batch record count + * - Acks arrive within ackTimeoutMs (5s) or session is retried + */ +class AsyncQueue { + private values: T[] = []; + private waiters: Array<(value: T) => void> = []; + + push(value: T): void { + const waiter = this.waiters.shift(); + if (waiter) { + waiter(value); + return; + } + this.values.push(value); + } + + async next(): Promise { + if (this.values.length > 0) { + return this.values.shift()!; + } + return new Promise((resolve) => { + this.waiters.push(resolve); + }); + } + + clear(): void { + this.values = []; + this.waiters = []; + } + + // Drain currently buffered values (non-blocking) and clear the buffer. + drain(): T[] { + const out = this.values; + this.values = []; + return out; + } +} + +/** + * New simplified inflight entry for the pump-based architecture. + * Each entry tracks a batch and its promise from the inner transport session. + */ +type InflightEntry = { + records: AppendRecord[]; + args?: Omit & { precalculatedSize?: number }; + expectedCount: number; + meteredBytes: number; + enqueuedAt: number; // Timestamp for timeout anchoring + innerPromise: Promise; // Promise from transport session + maybeResolve?: (result: AppendResult) => void; // Resolver for submit() callers +}; + +const DEFAULT_MAX_QUEUED_BYTES = 10 * 1024 * 1024; // 10 MiB default + +export class RetryAppendSession implements AppendSession, AsyncDisposable { + private readonly requestTimeoutMs: number; + private readonly maxQueuedBytes: number; + private readonly maxInflightBatches?: number; + private readonly retryConfig: Required & { + requestTimeoutMs: number; + }; + + private readonly inflight: InflightEntry[] = []; + private capacityWaiter?: () => void; // Single waiter (WritableStream writer lock) + + private session?: TransportAppendSession; + private queuedBytes = 0; + private pendingBytes = 0; + private consecutiveFailures = 0; + private currentAttempt = 0; + + private pumpPromise?: Promise; + private pumpStopped = false; + private closing = false; + private pumpWakeup?: () => void; + private closed = false; + private fatalError?: S2Error; + + private _lastAckedPosition?: AppendAck; + private acksController?: ReadableStreamDefaultController; + + public readonly readable: ReadableStream; + public readonly writable: WritableStream; + + /** + * If the session has failed, returns the original fatal error that caused + * the pump to stop. Returns undefined when the session has not failed. + */ + failureCause(): S2Error | undefined { + return this.fatalError; + } + + constructor( + private readonly generator: ( + options?: AppendSessionOptions, + ) => Promise, + private readonly sessionOptions?: AppendSessionOptions, + config?: RetryConfig, + ) { + this.retryConfig = { + ...DEFAULT_RETRY_CONFIG, + ...config, + }; + this.requestTimeoutMs = this.retryConfig.requestTimeoutMs; + this.maxQueuedBytes = + this.sessionOptions?.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES; + this.maxInflightBatches = this.sessionOptions?.maxInflightBatches; + + this.readable = new ReadableStream({ + start: (controller) => { + this.acksController = controller; + }, + }); + + this.writable = new WritableStream({ + write: async (chunk) => { + const recordsArray = Array.isArray(chunk.records) + ? chunk.records + : [chunk.records]; + + // Calculate metered size + let batchMeteredSize = 0; + for (const record of recordsArray) { + batchMeteredSize += meteredSizeBytes(record); + } + + // Wait for capacity (backpressure for writable only) + await this.waitForCapacity(batchMeteredSize); + + const args = { ...chunk } as Omit & { + precalculatedSize?: number; + }; + delete (args as any).records; + args.precalculatedSize = batchMeteredSize; + + // Submit without waiting for ack (writable doesn't need per-batch resolution) + const promise = this.submitInternal( + recordsArray, + args, + batchMeteredSize, + ); + promise.catch(() => { + // Swallow to avoid unhandled rejection; pump surfaces errors via readable stream + }); + }, + close: async () => { + await this.close(); + }, + abort: async (reason) => { + const error = new S2Error({ + message: `AppendSession aborted: ${reason}`, + status: 499, + }); + await this.abort(error); + }, + }); + } + + static async create( + generator: ( + options?: AppendSessionOptions, + ) => Promise, + sessionOptions?: AppendSessionOptions, + config?: RetryConfig, + ): Promise { + return new RetryAppendSession(generator, sessionOptions, config); + } + + /** + * Submit an append request. Returns a promise that resolves with the ack. + * This method does not block on capacity (only writable.write() does). + */ + async submit( + records: AppendRecord | AppendRecord[], + args?: Omit & { precalculatedSize?: number }, + ): Promise { + const recordsArray = Array.isArray(records) ? records : [records]; + + // Calculate metered size if not provided + let batchMeteredSize = args?.precalculatedSize ?? 0; + if (batchMeteredSize === 0) { + for (const record of recordsArray) { + batchMeteredSize += meteredSizeBytes(record); + } + } + + const result = await this.submitInternal( + recordsArray, + args, + batchMeteredSize, + ); + + // Convert discriminated union back to throw pattern for public API + if (result.ok) { + return result.value; + } else { + throw result.error; + } + } + + /** + * Internal submit that returns discriminated union. + * Creates inflight entry and starts pump if needed. + */ + private submitInternal( + records: AppendRecord[], + args: + | (Omit & { precalculatedSize?: number }) + | undefined, + batchMeteredSize: number, + ): Promise { + if (this.closed || this.closing) { + return Promise.resolve( + err(new S2Error({ message: "AppendSession is closed", status: 400 })), + ); + } + + // Check for fatal error (e.g., from abort()) + if (this.fatalError) { + debug( + "[SUBMIT] rejecting due to fatal error: %s", + this.fatalError.message, + ); + return Promise.resolve(err(this.fatalError)); + } + + // Create promise for submit() callers + return new Promise((resolve) => { + // Create inflight entry (innerPromise will be set when pump processes it) + const entry: InflightEntry & { __needsSubmit?: boolean } = { + records, + args, + expectedCount: records.length, + meteredBytes: batchMeteredSize, + enqueuedAt: Date.now(), + innerPromise: new Promise(() => {}), // Never-resolving placeholder + maybeResolve: resolve, + __needsSubmit: true, // Mark for pump to submit + }; + + debug( + "[SUBMIT] enqueueing %d records (%d bytes): inflight=%d->%d, queuedBytes=%d->%d", + records.length, + batchMeteredSize, + this.inflight.length, + this.inflight.length + 1, + this.queuedBytes, + this.queuedBytes + batchMeteredSize, + ); + + this.inflight.push(entry); + this.queuedBytes += batchMeteredSize; + + // Wake pump if it's sleeping + if (this.pumpWakeup) { + this.pumpWakeup(); + } + + // Start pump if not already running + this.ensurePump(); + }); + } + + /** + * Wait for capacity before allowing write to proceed (writable only). + */ + private async waitForCapacity(bytes: number): Promise { + debug( + "[CAPACITY] checking for %d bytes: queuedBytes=%d, pendingBytes=%d, maxQueuedBytes=%d, inflight=%d", + bytes, + this.queuedBytes, + this.pendingBytes, + this.maxQueuedBytes, + this.inflight.length, + ); + + // Check if we have capacity + while (true) { + // Check for fatal error before adding to pendingBytes + if (this.fatalError) { + debug( + "[CAPACITY] fatal error detected, rejecting: %s", + this.fatalError.message, + ); + throw this.fatalError; + } + + // Byte-based gating + if (this.queuedBytes + this.pendingBytes + bytes <= this.maxQueuedBytes) { + // Batch-based gating (if configured) + if ( + this.maxInflightBatches === undefined || + this.inflight.length < this.maxInflightBatches + ) { + debug( + "[CAPACITY] capacity available, adding %d to pendingBytes", + bytes, + ); + this.pendingBytes += bytes; + return; + } + } + + // No capacity - wait + // WritableStream enforces writer lock, so only one write can be blocked at a time + debug("[CAPACITY] no capacity, waiting for release"); + await new Promise((resolve) => { + this.capacityWaiter = resolve; + }); + debug("[CAPACITY] woke up, rechecking"); + } + } + + /** + * Release capacity and wake waiter if present. + */ + private releaseCapacity(bytes: number): void { + debug( + "[CAPACITY] releasing %d bytes: queuedBytes=%d->%d, pendingBytes=%d->%d, hasWaiter=%s", + bytes, + this.queuedBytes, + this.queuedBytes - bytes, + this.pendingBytes, + Math.max(0, this.pendingBytes - bytes), + !!this.capacityWaiter, + ); + this.queuedBytes -= bytes; + this.pendingBytes = Math.max(0, this.pendingBytes - bytes); + + // Wake single waiter + const waiter = this.capacityWaiter; + if (waiter) { + debug("[CAPACITY] waking waiter"); + this.capacityWaiter = undefined; + waiter(); + } + } + + /** + * Ensure pump loop is running. + */ + private ensurePump(): void { + if (this.pumpPromise || this.pumpStopped) { + return; + } + + this.pumpPromise = this.runPump().catch((e) => { + debug("pump crashed unexpectedly: %s", e); + // This should never happen - pump handles all errors internally + }); + } + + /** + * Main pump loop: processes inflight queue, handles acks, retries, and recovery. + */ + private async runPump(): Promise { + debug("pump started"); + + while (true) { + debug( + "[PUMP] loop: inflight=%d, queuedBytes=%d, pendingBytes=%d, closing=%s, pumpStopped=%s", + this.inflight.length, + this.queuedBytes, + this.pendingBytes, + this.closing, + this.pumpStopped, + ); + + // Check if we should stop + if (this.pumpStopped) { + debug("[PUMP] stopped by flag"); + return; + } + + // If closing and queue is empty, stop + if (this.closing && this.inflight.length === 0) { + debug("[PUMP] closing and queue empty, stopping"); + this.pumpStopped = true; + return; + } + + // If no entries, sleep and continue + if (this.inflight.length === 0) { + debug("[PUMP] no entries, sleeping 10ms"); + // Use interruptible sleep - can be woken by new submissions + await Promise.race([ + sleep(10), + new Promise((resolve) => { + this.pumpWakeup = resolve; + }), + ]); + this.pumpWakeup = undefined; + continue; + } + + // Get head entry (we know it exists because we checked length above) + const head = this.inflight[0]!; + debug( + "[PUMP] processing head: expectedCount=%d, meteredBytes=%d, enqueuedAt=%d", + head.expectedCount, + head.meteredBytes, + head.enqueuedAt, + ); + + // Ensure session exists + debug("[PUMP] ensuring session exists"); + await this.ensureSession(); + if (!this.session) { + // Session creation failed - will retry + debug("[PUMP] session creation failed, sleeping 100ms"); + await sleep(100); + continue; + } + + // Submit ALL entries that need submitting (enables HTTP/2 pipelining for S2S) + for (const entry of this.inflight) { + if (!entry.innerPromise || (entry as any).__needsSubmit) { + debug( + "[PUMP] submitting entry to inner session (%d records, %d bytes)", + entry.expectedCount, + entry.meteredBytes, + ); + entry.innerPromise = this.session.submit(entry.records, entry.args); + delete (entry as any).__needsSubmit; + } + } + + // Wait for head with timeout + debug("[PUMP] waiting for head result"); + const result = await this.waitForHead(head); + debug("[PUMP] got result: kind=%s", result.kind); + + if (result.kind === "timeout") { + // Ack timeout - fatal + const elapsed = Date.now() - head.enqueuedAt; + const error = new S2Error({ + message: `Request timeout after ${elapsed}ms (${head.expectedCount} records, ${head.meteredBytes} bytes, enqueued at ${new Date(head.enqueuedAt).toISOString()})`, + status: 408, + code: "REQUEST_TIMEOUT", + }); + debug("ack timeout for head entry: %s", error.message); + await this.abort(error); + return; + } + + // Promise settled + const appendResult = result.value; + + if (appendResult.ok) { + // Success! + const ack = appendResult.value; + debug("[PUMP] success, got ack", { ack }); + + // Invariant check: ack count matches batch count + const ackCount = Number(ack.end.seq_num) - Number(ack.start.seq_num); + if (ackCount !== head.expectedCount) { + const error = new S2Error({ + message: `Ack count mismatch: expected ${head.expectedCount}, got ${ackCount}`, + status: 500, + code: "INTERNAL_ERROR", + }); + debug("invariant violation: %s", error.message); + await this.abort(error); + return; + } + + // Invariant check: sequence numbers must be strictly increasing + if (this._lastAckedPosition) { + const prevEnd = BigInt(this._lastAckedPosition.end.seq_num); + const currentEnd = BigInt(ack.end.seq_num); + if (currentEnd <= prevEnd) { + const error = new S2Error({ + message: `Sequence number not strictly increasing: previous=${prevEnd}, current=${currentEnd}`, + status: 500, + code: "INTERNAL_ERROR", + }); + debug("invariant violation: %s", error.message); + await this.abort(error); + return; + } + } + + // Update last acked position + this._lastAckedPosition = ack; + + // Resolve submit() caller if present + if (head.maybeResolve) { + head.maybeResolve(ok(ack)); + } + + // Emit to readable stream + try { + this.acksController?.enqueue(ack); + } catch (e) { + debug("failed to enqueue ack: %s", e); + } + + // Remove from inflight and release capacity + debug( + "[PUMP] removing head from inflight, releasing %d bytes", + head.meteredBytes, + ); + this.inflight.shift(); + this.releaseCapacity(head.meteredBytes); + + // Reset consecutive failures on success + this.consecutiveFailures = 0; + this.currentAttempt = 0; + } else { + // Error result + const error = appendResult.error; + debug( + "[PUMP] error: status=%s, message=%s", + error.status, + error.message, + ); + + // Check if retryable + if (!isRetryable(error)) { + debug("error not retryable, aborting"); + await this.abort(error); + return; + } + + // Check policy compliance + if ( + this.retryConfig.appendRetryPolicy === "noSideEffects" && + !this.isAppendRetryAllowed(head) + ) { + debug("error not policy-compliant (noSideEffects), aborting"); + await this.abort(error); + return; + } + + // Check max attempts + if (this.currentAttempt >= this.retryConfig.maxAttempts) { + debug( + "max attempts reached (%d), aborting", + this.retryConfig.maxAttempts, + ); + const wrappedError = new S2Error({ + message: `Max retry attempts (${this.retryConfig.maxAttempts}) exceeded: ${error.message}`, + status: error.status, + code: error.code, + }); + await this.abort(wrappedError); + return; + } + + // Perform recovery + this.consecutiveFailures++; + this.currentAttempt++; + + debug( + "performing recovery (attempt %d/%d)", + this.currentAttempt, + this.retryConfig.maxAttempts, + ); + + await this.recover(); + } + } + } + + /** + * Wait for head entry's innerPromise with timeout. + * Returns either the settled result or a timeout indicator. + */ + private async waitForHead( + head: InflightEntry, + ): Promise<{ kind: "settled"; value: AppendResult } | { kind: "timeout" }> { + const deadline = head.enqueuedAt + this.requestTimeoutMs; + const remaining = Math.max(0, deadline - Date.now()); + + let timer: any; + const timeoutP = new Promise<{ kind: "timeout" }>((resolve) => { + timer = setTimeout(() => resolve({ kind: "timeout" }), remaining); + }); + + const settledP = head.innerPromise.then((result) => ({ + kind: "settled" as const, + value: result, + })); + + try { + return await Promise.race([settledP, timeoutP]); + } finally { + if (timer) clearTimeout(timer); + } + } + + /** + * Recover from transient error: recreate session and resubmit all inflight entries. + */ + private async recover(): Promise { + debug("starting recovery"); + + // Calculate backoff delay + const delay = calculateDelay( + this.consecutiveFailures - 1, + this.retryConfig.retryBackoffDurationMs, + ); + debug("backing off for %dms", delay); + await sleep(delay); + + // Teardown old session + if (this.session) { + try { + const closeResult = await this.session.close(); + if (!closeResult.ok) { + debug( + "error closing old session during recovery: %s", + closeResult.error.message, + ); + } + } catch (e) { + debug("exception closing old session: %s", e); + } + this.session = undefined; + } + + // Create new session + await this.ensureSession(); + if (!this.session) { + debug("failed to create new session during recovery"); + // Will retry on next pump iteration + return; + } + + // Store session in local variable to help TypeScript type narrowing + const session: TransportAppendSession = this.session; + + // Resubmit all inflight entries (replace their innerPromise) + debug("resubmitting %d inflight entries", this.inflight.length); + for (const entry of this.inflight) { + // Attach .catch to superseded promise to avoid unhandled rejection + entry.innerPromise.catch(() => {}); + + // Create new promise from new session + entry.innerPromise = session.submit(entry.records, entry.args); + } + + debug("recovery complete"); + } + + /** + * Check if append can be retried under noSideEffects policy. + */ + private isAppendRetryAllowed(entry: InflightEntry): boolean { + const args = entry.args; + if (!args) return false; + + // Allow retry if match_seq_num or fencing_token is set (idempotent) + return args.match_seq_num !== undefined || args.fencing_token !== undefined; + } + + /** + * Ensure session exists, creating it if necessary. + */ + private async ensureSession(): Promise { + if (this.session) { + return; + } + + try { + this.session = await this.generator(this.sessionOptions); + } catch (e) { + const error = s2Error(e); + debug("failed to create session: %s", error.message); + // Don't set this.session - will retry later + } + } + + /** + * Abort the session with a fatal error. + */ + private async abort(error: S2Error): Promise { + if (this.pumpStopped) { + return; // Already aborted + } + + debug("aborting session: %s", error.message); + + this.fatalError = error; + this.pumpStopped = true; + + // Resolve all inflight entries with error + for (const entry of this.inflight) { + if (entry.maybeResolve) { + entry.maybeResolve(err(error)); + } + } + this.inflight.length = 0; + this.queuedBytes = 0; + this.pendingBytes = 0; + + // Error the readable stream + try { + this.acksController?.error(error); + } catch (e) { + debug("failed to error acks controller: %s", e); + } + + // Wake capacity waiter to unblock any pending writer + if (this.capacityWaiter) { + this.capacityWaiter(); + this.capacityWaiter = undefined; + } + + // Close inner session + if (this.session) { + try { + await this.session.close(); + } catch (e) { + debug("error closing session during abort: %s", e); + } + this.session = undefined; + } + } + + /** + * Close the append session. + * Waits for all pending appends to complete before resolving. + * Does not interrupt recovery - allows it to complete. + */ + async close(): Promise { + if (this.closed) { + if (this.fatalError) { + throw this.fatalError; + } + return; + } + + debug("close requested"); + this.closing = true; + + // Wake pump if it's sleeping so it can check closing flag + if (this.pumpWakeup) { + this.pumpWakeup(); + } + + // Wait for pump to stop (drains inflight queue, including through recovery) + if (this.pumpPromise) { + await this.pumpPromise; + } + + // Close inner session + if (this.session) { + try { + const result = await this.session.close(); + if (!result.ok) { + debug("error closing inner session: %s", result.error.message); + } + } catch (e) { + debug("exception closing inner session: %s", e); + } + this.session = undefined; + } + + // Close readable stream + try { + this.acksController?.close(); + } catch (e) { + debug("error closing acks controller: %s", e); + } + + this.closed = true; + + // If fatal error occurred, throw it + if (this.fatalError) { + throw this.fatalError; + } + + debug("close complete"); + } + + async [Symbol.asyncDispose](): Promise { + await this.close(); + } + + /** + * Get a stream of acknowledgements for appends. + */ + acks(): AcksStream { + return this.readable as AcksStream; + } + + /** + * Get the last acknowledged position. + */ + lastAckedPosition(): AppendAck | undefined { + return this._lastAckedPosition; + } +} diff --git a/src/lib/retry.ts.old b/src/lib/retry.ts.old new file mode 100644 index 0000000..94ffec5 --- /dev/null +++ b/src/lib/retry.ts.old @@ -0,0 +1,1408 @@ +import createDebug from "debug"; +import type { RetryConfig } from "../common.js"; +import { S2Error, s2Error, withS2Error } from "../error.js"; +import type { AppendAck, StreamPosition } from "../generated/index.js"; +import { meteredSizeBytes } from "../utils.js"; +import type { + AcksStream, + AppendArgs, + AppendRecord, + AppendSession, + AppendSessionOptions, + ReadArgs, + ReadRecord, + ReadSession, +} from "./stream/types.js"; +import type { AppendResult, CloseResult } from "./result.js"; +import { ok, err, okClose, errClose } from "./result.js"; + +const debug = createDebug("s2:retry"); + +/** + * Default retry configuration. + */ +export const DEFAULT_RETRY_CONFIG: Required & { + requestTimeoutMs: number; +} = { + maxAttempts: 3, + retryBackoffDurationMs: 100, + appendRetryPolicy: "noSideEffects", + requestTimeoutMs: 5000, // 5 seconds +}; + +const RETRYABLE_STATUS_CODES = new Set([ + 408, // request_timeout + 429, // too_many_requests + 500, // internal_server_error + 502, // bad_gateway + 503, // service_unavailable +]); + +/** + * Determines if an error should be retried based on its characteristics. + * 400-level errors (except 408, 429) are non-retryable validation/client errors. + */ +export function isRetryable(error: S2Error): boolean { + if (!error.status) return false; + + // Explicit retryable codes (including some 4xx like 408, 429) + if (RETRYABLE_STATUS_CODES.has(error.status)) { + return true; + } + + // 400-level errors are generally non-retryable (validation, bad request) + if (error.status >= 400 && error.status < 500) { + return false; + } + + return false; +} + +/** + * Calculates the delay before the next retry attempt using exponential backoff. + */ +export function calculateDelay(attempt: number, baseDelayMs: number): number { + // Exponential backoff: baseDelay * (2 ^ attempt) + const delay = baseDelayMs * Math.pow(2, attempt); + // Add jitter: random value between 0 and delay + const jitter = Math.random() * delay; + return Math.floor(delay + jitter); +} + +/** + * Sleeps for the specified duration. + */ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Executes an async function with automatic retry logic for transient failures. + * + * @param retryConfig Retry configuration (max attempts, backoff duration) + * @param fn The async function to execute + * @returns The result of the function + * @throws The last error if all retry attempts are exhausted + */ +export async function withRetries( + retryConfig: RetryConfig | undefined, + fn: () => Promise, + isPolicyCompliant: (config: RetryConfig, error: S2Error) => boolean = () => + true, +): Promise { + const config = { + ...DEFAULT_RETRY_CONFIG, + ...retryConfig, + }; + + // If maxAttempts is 0, don't retry at all + if (config.maxAttempts === 0) { + debug("maxAttempts is 0, retries disabled"); + return fn(); + } + + let lastError: S2Error | undefined = undefined; + + for (let attempt = 0; attempt <= config.maxAttempts; attempt++) { + try { + const result = await fn(); + if (attempt > 0) { + debug("succeeded after %d retries", attempt); + } + return result; + } catch (error) { + // withRetry only handles S2Errors (withS2Error should be called first) + if (!(error instanceof S2Error)) { + debug("non-S2Error thrown, rethrowing immediately: %s", error); + throw error; + } + + lastError = error; + + // Don't retry if this is the last attempt + if (attempt === config.maxAttempts) { + debug("max attempts exhausted, throwing error"); + break; + } + + // Check if error is retryable + if (!isPolicyCompliant(config, lastError) || !isRetryable(lastError)) { + debug("error not retryable, throwing immediately"); + throw error; + } + + // Calculate delay and wait before retrying + const delay = calculateDelay(attempt, config.retryBackoffDurationMs); + debug( + "retryable error, backing off for %dms, status=%s", + delay, + error.status, + ); + await sleep(delay); + } + } + + throw lastError; +} +export class RetryReadSession + extends ReadableStream> + implements ReadSession +{ + private _nextReadPosition: StreamPosition | undefined = undefined; + + private _recordsRead: number = 0; + private _bytesRead: number = 0; + + static async create( + generator: (args: ReadArgs) => Promise>, + args: ReadArgs = {}, + config?: RetryConfig, + ) { + return new RetryReadSession(args, generator, config); + } + + private constructor( + args: ReadArgs, + generator: (args: ReadArgs) => Promise>, + config?: RetryConfig, + ) { + const retryConfig = { + ...DEFAULT_RETRY_CONFIG, + ...config, + }; + let session: ReadSession | undefined = undefined; + super({ + start: async (controller) => { + let nextArgs = { ...args }; + let attempt = 0; + + while (true) { + try { + session = await generator(nextArgs); + const reader = session.getReader(); + + while (true) { + const { done, value } = await reader.read(); + attempt = 0; + if (done) { + break; + } + this._nextReadPosition = { + seq_num: value.seq_num + 1, + timestamp: value.timestamp, + }; + this._recordsRead++; + this._bytesRead += meteredSizeBytes(value); + + controller.enqueue(value); + } + reader.releaseLock(); + break; + } catch (e) { + let error = s2Error(e); + if (isRetryable(error) && attempt < retryConfig.maxAttempts) { + if (this._nextReadPosition) { + nextArgs.seq_num = this._nextReadPosition.seq_num; + } + if (nextArgs.count) { + nextArgs.count = + this._recordsRead === undefined + ? nextArgs.count + : nextArgs.count - this._recordsRead; + } + if (nextArgs.bytes) { + nextArgs.bytes = + this._bytesRead === undefined + ? nextArgs.bytes + : nextArgs.bytes - this._bytesRead; + } + // TODO also correct wait + const delay = calculateDelay( + attempt, + retryConfig.retryBackoffDurationMs, + ); + debug("will retry after %dms, status=%s", delay, error.status); + await sleep(delay); + attempt++; + continue; + } + + debug("error in retry loop: %s", e); + throw error; + } + } + + controller.close(); + }, + cancel: async () => { + session?.cancel(); + }, + }); + } + + async [Symbol.asyncDispose]() { + await this.cancel("disposed"); + } + + // Polyfill for older browsers / Node.js environments + [Symbol.asyncIterator](): AsyncIterableIterator> { + const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; + if (typeof fn === "function") return fn.call(this); + const reader = this.getReader(); + return { + next: async () => { + const r = await reader.read(); + if (r.done) { + reader.releaseLock(); + return { done: true, value: undefined }; + } + return { done: false, value: r.value }; + }, + throw: async (e) => { + await reader.cancel(e); + reader.releaseLock(); + return { done: true, value: undefined }; + }, + return: async () => { + await reader.cancel("done"); + reader.releaseLock(); + return { done: true, value: undefined }; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + } + + lastObservedTail(): StreamPosition | undefined { + return undefined; + } + + nextReadPosition(): StreamPosition | undefined { + return undefined; + } +} + +/** + * RetryAppendSession wraps an underlying AppendSession with automatic retry logic. + * + * Architecture: + * - All writes (submit() and writable.write()) are serialized through inflightQueue + * - inflightQueue tracks batches that have been submitted but not yet acked + * - Background ack reader consumes acks and matches them FIFO with inflightQueue + * - On error, _initSession() recreates session and re-transmits all inflightQueue batches + * - Ack timeout is fatal: if no ack arrives within the timeout window, + * the session aborts and rejects queued writers + * + * Flow for a successful append: + * 1. submit(records) adds batch to inflightQueue with promise resolvers + * 2. Calls underlying session.submit() to send batch + * 3. Background reader receives ack, validates record count + * 4. Resolves promise, removes from inflightQueue, forwards ack to user + * + * Flow for a failed append: + * 1. submit(records) adds batch to inflightQueue + * 2. Calls underlying session.submit() which fails + * 3. Checks if retryable (status code, retry policy, idempotency) + * 4. Calls _initSession() which closes old session, creates new session + * 5. _initSession() re-transmits ALL batches in inflightQueue (recovery) + * 6. Background reader receives acks for recovered batches + * 7. Original submit() call's promise is resolved by background reader + * + * Invariants: + * - Exactly one ack per batch in FIFO order + * - Ack record count matches batch record count + * - Acks arrive within ackTimeoutMs (5s) or session is retried + */ +class AsyncQueue { + private values: T[] = []; + private waiters: Array<(value: T) => void> = []; + + push(value: T): void { + const waiter = this.waiters.shift(); + if (waiter) { + waiter(value); + return; + } + this.values.push(value); + } + + async next(): Promise { + if (this.values.length > 0) { + return this.values.shift()!; + } + return new Promise((resolve) => { + this.waiters.push(resolve); + }); + } + + clear(): void { + this.values = []; + this.waiters = []; + } + + // Drain currently buffered values (non-blocking) and clear the buffer. + drain(): T[] { + const out = this.values; + this.values = []; + return out; + } +} + +/** + * New simplified inflight entry for the pump-based architecture. + * Each entry tracks a batch and its promise from the inner transport session. + */ +type InflightEntry = { + records: AppendRecord[]; + args?: Omit & { precalculatedSize?: number }; + expectedCount: number; + meteredBytes: number; + enqueuedAt: number; // Timestamp for timeout anchoring + innerPromise: Promise; // Promise from transport session + maybeResolve?: (result: AppendResult) => void; // Resolver for submit() callers +}; + +const DEFAULT_MAX_QUEUED_BYTES = 10 * 1024 * 1024; // 10 MiB default + +export class RetryAppendSession implements AppendSession, AsyncDisposable { + private readonly ackTimeoutMs = 10000; + private readonly maxQueuedBytes: number; + private readonly retryConfig: Required; + + private readonly notificationQueue = new AsyncQueue(); + private readonly inflight: InflightEntry[] = []; + private readonly capacityWaiters: Array<() => void> = []; + + private session: AppendSession | undefined = undefined; + private sessionWriter?: WritableStreamDefaultWriter; + private sessionReader?: ReadableStreamDefaultReader; + + private queuedBytes = 0; + private pendingBytes = 0; + private consecutiveFailures = 0; + + private pumpPromise?: Promise; + private pumpStopped = false; + private closePromise?: Promise; + private closeResolve?: () => void; + private closeReject?: (error: S2Error) => void; + private closing = false; + private pumpError?: S2Error; + private readerToken?: object; + private readerTokenId?: number; + private readerTokenGen: number = 0; + private readonly readerTokenIds = new WeakMap(); + private ackTimeoutHandle?: ReturnType; + private ackTimeoutToken?: object; + private ackTimeoutHead?: InflightEntry; + // removed recoveryPromise in favor of explicit flags + private closed = false; + + // Recovery gating + private pausedForRecovery: boolean = false; + private recoveryBarrierLeft: number = 0; + private backlog: InflightEntry[] = []; + private recovering: boolean = false; + private restartRequested: boolean = false; + private maxInflightBatches?: number; + private reservedBatches: number = 0; + + private _lastAckedPosition?: AppendAck; + private acksController?: ReadableStreamDefaultController; + + public readonly readable: ReadableStream; + public readonly writable: WritableStream; + + /** + * If the session has failed, returns the original fatal error that caused + * the pump to stop. Returns undefined when the session has not failed. + */ + failureCause(): S2Error | undefined { + return this.pumpError; + } + + constructor( + private readonly generator: ( + options?: AppendSessionOptions, + ) => Promise, + private readonly sessionOptions?: AppendSessionOptions, + config?: RetryConfig, + ) { + this.retryConfig = { + ...DEFAULT_RETRY_CONFIG, + ...config, + }; + this.maxQueuedBytes = + this.sessionOptions?.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES; + this.maxInflightBatches = this.sessionOptions?.maxInflightBatches; + + this.readable = new ReadableStream({ + start: (controller) => { + this.acksController = controller; + }, + }); + + this.writable = new WritableStream({ + write: async (chunk) => { + const recordsArray = Array.isArray(chunk.records) + ? chunk.records + : [chunk.records]; + const args = { ...chunk } as Omit & { + precalculatedSize?: number; + }; + delete (args as any).records; + const { ackPromise, enqueuedPromise } = await this.enqueueBatch( + recordsArray, + args, + ); + ackPromise.catch(() => { + // Swallow to avoid unhandled rejection; pump already surfaces the error. + }); + return enqueuedPromise; + }, + close: async () => { + await this.close(); + }, + abort: async (reason) => { + await this.abort(reason); + }, + }); + } + + static async create( + generator: ( + options?: AppendSessionOptions, + ) => Promise, + sessionOptions?: AppendSessionOptions, + config?: RetryConfig, + ): Promise { + return new RetryAppendSession(generator, sessionOptions, config); + } + + private ensurePump(): void { + if (this.pumpPromise) { + return; + } + this.pumpPromise = this.runPump(); + } + + private async runPump(): Promise { + try { + while (true) { + if (this.pumpStopped) { + break; + } + if (this.shouldAttemptClose()) { + await this.finishClose(); + continue; + } + const notification = await this.notificationQueue.next(); + if (notification.type === "stop") { + break; + } + await this.processNotification(notification); + } + } catch (error) { + this.setPumpError(s2Error(error)); + } finally { + this.closed = true; + this.pumpStopped = true; + this.resolveClosePromise(); + } + } + + private shouldAttemptClose(): boolean { + const ok = + this.closing && + !this.pumpStopped && + this.inflight.length === 0 && + !this.pausedForRecovery && + !this.recovering && + this.backlog.length === 0; + if (this.closing) { + debug( + "shouldAttemptClose? %s (inflight=%d paused=%s recovering=%s backlog=%d)", + ok, + this.inflight.length, + this.pausedForRecovery, + this.recovering, + this.backlog.length, + ); + } + return ok; + } + + private async processNotification(notification: Notification): Promise { + const tokId = (notification as any).token + ? this.readerTokenIds.get((notification as any).token) + : undefined; + debug( + "process: inflight=%d type=%s tokenId=%s", + this.inflight.length, + notification.type, + tokId ?? "none", + ); + switch (notification.type) { + case "batch": + await this.handleBatch(notification.entry); + break; + case "ack": + if (notification.token && notification.token !== this.readerToken) { + break; + } + await this.handleAck(notification.ack); + break; + case "error": + if (notification.token && notification.token !== this.readerToken) { + break; + } + await this.startRecovery(notification.error); + break; + case "close": + debug("close"); + await this.handleClose(notification); + break; + case "abort": + debug("abort"); + await this.handleAbort(notification.error); + break; + case "stop": + return; + } + } + + private async handleBatch(entry: InflightEntry): Promise { + this.pendingBytes = Math.max(0, this.pendingBytes - entry.meteredBytes); + // Consume a reserved batch slot if batch-based gating is enabled + if (this.maxInflightBatches && this.maxInflightBatches > 0 && this.reservedBatches > 0) { + this.reservedBatches -= 1; + } + + // If recovering (gated), backlog the entry and do not send yet. + if (this.pausedForRecovery) { + this.backlog.push(entry); + this.queuedBytes += entry.meteredBytes; + // Do not arm head timer here; we only arm when actually sending. + this.releaseCapacity(); + return; + } + + // Normal path: inflight + send + this.inflight.push(entry); + this.queuedBytes += entry.meteredBytes; + this.releaseCapacity(); + + try { + await this.ensureSession(); + await this.sendEntry(entry); + } catch (error) { + const err = s2Error(error); + entry.sent = false; + debug("Error sending batch: %s", err.message); + // Respect append retry policy: only retry when allowed + if (!isRetryable(err) || !this.isAppendRetryAllowed(entry)) { + this.removeInflightEntry(entry); + entry.reject(err); + this.setPumpError(err); + return; + } + this.notificationQueue.push({ type: "error", error: err }); + } + } + + private async handleAck(ack: AppendAck): Promise { + if (this.inflight.length === 0) { + const fatal = new S2Error({ + message: "Invariant violation: received ack with empty inflight queue", + status: 500, + }); + this.setPumpError(fatal); + return; + } + + const entry = this.inflight.shift()!; + const actualCount = ack.end.seq_num - ack.start.seq_num; + if (actualCount !== entry.expectedCount) { + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + this.releaseCapacity(); + const error = new S2Error({ + message: `Invariant violation: Ack record count mismatch. Expected ${entry.expectedCount}, got ${actualCount}`, + status: 500, + }); + entry.reject(error); + this.setPumpError(error); + return; + } + + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + this.releaseCapacity(); + + this._lastAckedPosition = ack; + this.consecutiveFailures = 0; + + entry.resolve(ack); + if (this.acksController) { + try { + this.acksController.enqueue(ack); + } catch (error) { + debug("Failed to enqueue ack: %s", error); + } + } + + this.startAckTimerForHead(); + + // If recovering, count down acks from the snapshot; when drained, flush backlog and resume. + if (this.pausedForRecovery && this.recoveryBarrierLeft > 0) { + debug( + "recovery ack: remaining=%d inflight_after=%d", + this.recoveryBarrierLeft - 1, + this.inflight.length, + ); + this.recoveryBarrierLeft -= 1; + if (this.recoveryBarrierLeft === 0) { + debug("recovery barrier drained; flushing backlog size=%d", this.backlog.length); + await this.flushBacklog(); + this.pausedForRecovery = false; + this.recovering = false; + debug("recovery complete; resuming writers (waiters=%d)", this.capacityWaiters.length); + // Wake blocked writers + while (this.capacityWaiters.length > 0) { + const resolve = this.capacityWaiters.shift(); + resolve?.(); + } + await this.maybeFinishClose(); + } + } + + await this.maybeFinishClose(); + } + + private async handleClose(_: CloseNotification): Promise { + this.closing = true; + await this.maybeFinishClose(); + } + + private async handleAbort(error: S2Error): Promise { + if (this.pumpStopped) { + return; + } + this.closing = true; + // Cache the original fatal cause if not already recorded + if (!this.pumpError) { + this.pumpError = error; + } + this.failAll(error); + // Reject any backlogged entries that were never sent + if (this.backlog.length > 0) { + for (const entry of this.backlog) { + try { + entry.reject(error); + } catch {} + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + } + this.backlog = []; + } + // Drain and reject any queued-but-not-processed batches. + const drained = this.notificationQueue.drain(); + for (const n of drained) { + if ((n as any).type === "batch") { + const bn = n as any; + this.pendingBytes = Math.max(0, this.pendingBytes - bn.entry.meteredBytes); + try { + bn.entry.reject(error); + } catch {} + } + } + await this.teardownSession(); + // Ensure timeout continues across recovery even if we can't resend immediately. + this.startAckTimerForHead(); + if (this.acksController) { + try { + this.acksController.error(error); + } catch (err) { + debug("Error signaling acks controller during abort: %s", err); + } + } + // Wake all capacity waiters to ensure no writers remain blocked. + while (this.capacityWaiters.length > 0) { + const resolve = this.capacityWaiters.shift(); + try { + resolve?.(); + } catch {} + } + this.stopPump(error); + } + + private async ensureSession(): Promise { + if (this.session && this.sessionWriter && this.sessionReader) { + return; + } + this.session = await this.generator(this.sessionOptions); + this.sessionWriter = this.session.writable.getWriter(); + this.sessionReader = this.session.acks().getReader(); + this.startAckReader(this.sessionReader); + } + + private startAckReader(reader: ReadableStreamDefaultReader): void { + const token = {}; + const id = ++this.readerTokenGen; + this.readerToken = token; + this.readerTokenId = id; + this.readerTokenIds.set(token, id); + debug("reader[%d]: start", id); + + (async () => { + try { + while (true) { + const { done, value } = await reader.read(); + if (this.readerToken !== token) { + break; + } + if (done) { + debug("reader[%d]: done -> enqueue error closed", id); + this.notificationQueue.push({ + type: "error", + error: new S2Error({ + message: "AppendSession reader closed unexpectedly", + status: 500, + }), + token, + }); + break; + } + debug("reader[%d]: ack", id); + this.notificationQueue.push({ type: "ack", ack: value!, token }); + } + } catch (error) { + if (this.readerToken === token) { + debug("reader[%d]: error %s", id, (error as any)?.message ?? String(error)); + this.notificationQueue.push({ + type: "error", + error: s2Error(error), + token, + }); + } + } + })(); + } + + private async sendEntry(entry: InflightEntry): Promise { + if (!this.sessionWriter) { + throw new S2Error({ message: "AppendSession is not ready", status: 500 }); + } + const { precalculatedSize, ...rest } = entry.args ?? {}; + const payload = { + ...(rest as Omit), + records: entry.records, + } as AppendArgs; + + const firstSend = !entry.sent; + entry.sent = true; + if (firstSend) { + entry.enqueuedAt = Date.now(); + } + await this.sessionWriter.write(payload); + if (this.inflight.length > 0 && this.inflight[0] === entry) { + this.startAckTimerForHead(firstSend); + } + } + + private async startRecovery(reason: S2Error): Promise { + debug("Starting recovery due to: %s", reason.message); + if (this.pumpError) return; + if (this.closing && this.inflight.length === 0) { + await this.maybeFinishClose(); + return; + } + // Only retry errors that are retryable by status and policy + if (!isRetryable(reason)) { + this.setPumpError(reason); + return; + } + if ( + (this.retryConfig.appendRetryPolicy ?? "noSideEffects") === + "noSideEffects" && + this.inflight.some((e) => !this.isAppendRetryAllowed(e)) + ) { + // Abort via notification so that backlog and queued requests are rejected too + this.notificationQueue.push({ type: "abort", error: reason }); + return; + } + if (!this.pausedForRecovery) { + this.pausedForRecovery = true; + this.recoveryBarrierLeft = this.inflight.length; + debug( + "pausing enqueues for recovery; barrier=%d inflight=%d backlog=%d", + this.recoveryBarrierLeft, + this.inflight.length, + this.backlog.length, + ); + } + if (!this.recovering) { + this.recovering = true; + void this.runRecoveryLoop(); + } else { + this.restartRequested = true; + } + } + + private async runRecoveryLoop(): Promise { + while (this.recovering) { + this.consecutiveFailures += 1; + debug("Recovering RetryAppendSession (attempt %d)", this.consecutiveFailures); + if (this.pumpStopped || this.closing) return; + + await this.teardownSession(true); + if (this.consecutiveFailures > this.retryConfig.maxAttempts) { + const error = new S2Error({ + message: `Max retry attempts (${this.retryConfig.maxAttempts}) exceeded`, + status: 500, + }); + this.failAll(error); + this.stopPump(error); + return; + } + + const delay = calculateDelay( + this.consecutiveFailures - 1, + this.retryConfig.retryBackoffDurationMs, + ); + await sleep(delay); + if (this.pumpStopped || this.closing) return; + + try { + this.session = await this.generator(this.sessionOptions); + this.sessionWriter = this.session.writable.getWriter(); + this.sessionReader = this.session.acks().getReader(); + this.startAckReader(this.sessionReader); + } catch (e) { + this.restartRequested = true; + } + + debug("resending inflight batches (%d)", this.inflight.length); + for (const entry of this.inflight) { + try { + if ( + (this.retryConfig.appendRetryPolicy ?? "noSideEffects") === + "noSideEffects" && + !this.isAppendRetryAllowed(entry) + ) { + // Abort and surface a policy error; do not continue resending. + this.notificationQueue.push({ + type: "abort", + error: new S2Error({ + message: "Append retry not allowed by policy", + status: 500, + }), + }); + this.restartRequested = false; + return; + } + await this.ensureSession(); + await this.sendEntry(entry); + } catch (error) { + this.restartRequested = true; + break; + } + } + debug("resending complete"); + + if (this.restartRequested) { + this.restartRequested = false; + continue; + } + + // If there are no inflight entries to wait for, flush backlog immediately and resume. + if (this.pausedForRecovery && this.recoveryBarrierLeft === 0) { + debug("recovery barrier is zero; flushing backlog now"); + await this.flushBacklog(); + this.pausedForRecovery = false; + this.recovering = false; + this.consecutiveFailures = 0; + // Wake any blocked writers. + while (this.capacityWaiters.length > 0) { + const resolve = this.capacityWaiters.shift(); + resolve?.(); + } + await this.maybeFinishClose(); + return; + } + + while (this.pausedForRecovery && this.recoveryBarrierLeft > 0 && !this.restartRequested && !this.pumpError && !this.closing) { + await sleep(5); + } + if (this.restartRequested) { + this.restartRequested = false; + continue; + } + if (!this.pausedForRecovery && this.recoveryBarrierLeft === 0) { + this.recovering = false; + this.consecutiveFailures = 0; + await this.maybeFinishClose(); + return; + } + } + } + + // performRecovery and sendUnsentInflightEntries removed in favor of explicit runRecoveryLoop + gating + + private async teardownSession(preserveAckTimer?: boolean): Promise { + if (!preserveAckTimer) { + this.clearAckTimeout(); + } + if (this.sessionReader) { + try { + if (this.readerTokenId !== undefined) { + debug("teardown: cancel reader[%d]", this.readerTokenId); + } + await this.sessionReader.cancel(); + } catch (error) { + debug("Error cancelling ack reader: %s", error); + } + } + this.sessionReader = undefined; + this.readerToken = undefined; + this.readerTokenId = undefined; + + if (this.sessionWriter) { + try { + this.sessionWriter.releaseLock(); + } catch (error) { + debug("Error releasing session writer: %s", error); + } + } + this.sessionWriter = undefined; + + if (this.session) { + try { + await this.session.close(); + } catch (error) { + debug("Error closing underlying session: %s", error); + } + } + this.session = undefined; + } + + private startAckTimerForHead(force: boolean = false): void { + const head = this.inflight[0]; + if (!head) { + // No inflight; clear any existing timer + this.clearAckTimeout(); + this.ackTimeoutHead = undefined; + return; + } + // If the timer is already armed for this head, do nothing to avoid + // cancel/reschedule loops that could postpone the callback under load. + if (!force && this.ackTimeoutHandle && this.ackTimeoutHead === head) { + return; + } + this.clearAckTimeout(); + const delay = Math.max(0, head.enqueuedAt + this.ackTimeoutMs - Date.now()); + debug( + "arm head timer: force=%s delayMs=%d inflight=%d", + force, + delay, + this.inflight.length, + ); + const token = {}; + this.ackTimeoutHead = head; + this.ackTimeoutToken = token; + this.ackTimeoutHandle = setTimeout(() => { + if (this.ackTimeoutToken !== token) { + return; + } + this.ackTimeoutHead = undefined; + // Ack timeout should abort the session rather than be retried. + this.notificationQueue.push({ + type: "abort", + error: new S2Error({ + message: + "Ack timeout: no acknowledgement received within timeout period", + status: 408, + }), + }); + }, delay); + } + + private clearAckTimeout(): void { + if (this.ackTimeoutHandle) { + clearTimeout(this.ackTimeoutHandle); + this.ackTimeoutHandle = undefined; + this.ackTimeoutToken = undefined; + this.ackTimeoutHead = undefined; + } + } + + private async flushBacklog(): Promise { + if (this.backlog.length === 0) return; + debug("flushing backlog (%d)", this.backlog.length); + const backlog = this.backlog; + this.backlog = []; + for (const entry of backlog) { + try { + this.inflight.push(entry); + await this.ensureSession(); + await this.sendEntry(entry); + } catch (error) { + // On send failure, push error to restart recovery and requeue remaining entries + const idx = backlog.indexOf(entry); + if (idx >= 0 && idx + 1 < backlog.length) { + const remaining = backlog.slice(idx + 1); + // Put back remaining at front so they are not lost + this.backlog.unshift(...remaining); + } + this.notificationQueue.push({ type: "error", error: s2Error(error) }); + return; + } + } + } + + private async drainInflight(): Promise { + if (this.inflight.length === 0) { + return; + } + const timeoutMs = 30000; + const start = Date.now(); + while (this.inflight.length > 0) { + if (Date.now() - start > timeoutMs) { + throw new S2Error({ + message: "Close timeout: pending acks not received", + status: 408, + }); + } + await sleep(50); + } + } + + private failAll(error: S2Error): void { + for (const entry of this.inflight) { + entry.reject(error); + } + this.inflight.length = 0; + this.queuedBytes = 0; + this.pendingBytes = 0; + this.releaseCapacity(); + } + + private setPumpError(error: S2Error): void { + if (this.pumpError) { + return; + } + this.pumpError = error; + this.failAll(error); + // Also reject any backlogged entries and queued-but-not-processed batches + if (this.backlog.length > 0) { + for (const entry of this.backlog) { + try { + entry.reject(error); + } catch {} + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + } + this.backlog = []; + } + const drained = this.notificationQueue.drain(); + for (const n of drained) { + if ((n as any).type === "batch") { + const bn = n as any; + this.pendingBytes = Math.max(0, this.pendingBytes - bn.entry.meteredBytes); + try { + bn.entry.reject(error); + } catch {} + } + } + if (this.acksController) { + try { + this.acksController.error(error); + } catch (err) { + debug("Error signaling acks controller: %s", err); + } + } + this.stopPump(error); + } + + private releaseCapacity(): void { + // Wake a single waiter; the waiter will re-check capacity under either + // byte-based or batch-based gating. + const resolve = this.capacityWaiters.shift(); + resolve?.(); + } + + private async finishClose(): Promise { + if (this.pumpStopped) { + return; + } + try { + debug("finishClose: rejecting inflight=%d backlog=%d", this.inflight.length, this.backlog.length); + // Reject any backlogged entries that were never sent + if (this.backlog.length > 0) { + for (const entry of this.backlog) { + try { + entry.reject(new S2Error({ message: "AppendSession is closed", status: 400 })); + } catch {} + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + } + this.backlog = []; + } + // Drain and reject any queued-but-not-processed batches. + const drained = this.notificationQueue.drain(); + let drainedBatches = 0; + for (const n of drained) { + if (n.type === "batch") { + drainedBatches += 1; + // Adjust pendingBytes since handleBatch won't run for these entries anymore. + this.pendingBytes = Math.max( + 0, + this.pendingBytes - n.entry.meteredBytes, + ); + try { + n.entry.reject( + new S2Error({ message: "AppendSession is closed", status: 400 }), + ); + } catch {} + } + } + if (this.maxInflightBatches && this.maxInflightBatches > 0 && drainedBatches > 0) { + this.reservedBatches = Math.max(0, this.reservedBatches - drainedBatches); + } + + await this.teardownSession(); + if (!this.pumpError && this.acksController) { + try { + this.acksController.close(); + } catch (error) { + debug("Error closing acks controller: %s", error); + } + } + debug("finishClose: stopping pump"); + // Wake all capacity waiters to unblock any writers waiting on backpressure. + while (this.capacityWaiters.length > 0) { + const resolve = this.capacityWaiters.shift(); + try { + resolve?.(); + } catch {} + } + this.stopPump(); + } catch (error) { + this.stopPump(s2Error(error)); + } + } + + private stopPump(error?: S2Error): void { + if (this.pumpStopped) { + return; + } + this.pumpStopped = true; + if (error && !this.pumpError) { + this.pumpError = error; + } + if (error) { + this.closeReject?.(error); + } else { + this.closeResolve?.(); + } + this.closeResolve = undefined; + this.closeReject = undefined; + this.notificationQueue.push({ type: "stop" }); + } + + private resolveClosePromise(): void { + if (this.closePromise && !this.pumpError) { + this.closeResolve?.(); + } + this.closeResolve = undefined; + this.closeReject = undefined; + } + + private async maybeFinishClose(): Promise { + if (!this.closing || this.pumpStopped) { + return; + } + if (this.inflight.length > 0 || this.pausedForRecovery || this.recovering || this.backlog.length > 0) { + return; + } + await this.finishClose(); + } + + private async waitForCapacity(bytes: number): Promise { + while (!this.closing && !this.pumpError) { + if (this.pausedForRecovery) { + debug( + "waitForCapacity: paused=true (inflight=%d backlog=%d); writer blocked", + this.inflight.length, + this.backlog.length, + ); + await new Promise((resolve) => { + this.capacityWaiters.push(resolve); + }); + continue; + } + if (this.maxInflightBatches && this.maxInflightBatches > 0) { + // Batch-based gating: reserve a slot if available + if (this.inflight.length + this.backlog.length + this.reservedBatches < this.maxInflightBatches) { + this.reservedBatches += 1; + return; + } + } else { + // Byte-based gating + if (this.queuedBytes + this.pendingBytes + bytes <= this.maxQueuedBytes) { + this.pendingBytes += bytes; + return; + } + } + await new Promise((resolve) => { + this.capacityWaiters.push(resolve); + }); + } + if (this.pumpError) { + throw this.pumpError; + } + throw new S2Error({ message: "AppendSession is closed", status: 400 }); + } + + private cloneRecords(records: AppendRecord[]): AppendRecord[] { + return records.map((record) => { + const cloned: AppendRecord = { ...record }; + if (record.body instanceof Uint8Array) { + cloned.body = record.body.slice(); + } + if (record.headers) { + if (Array.isArray(record.headers)) { + cloned.headers = record.headers.map((h) => [ + ...h, + ]) as AppendRecord["headers"]; + } else { + cloned.headers = { ...record.headers } as AppendRecord["headers"]; + } + } + return cloned; + }); + } + + private removeInflightEntry(entry: InflightEntry): void { + const idx = this.inflight.indexOf(entry); + if (idx !== -1) { + this.inflight.splice(idx, 1); + } + this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); + this.releaseCapacity(); + } + + private isAppendRetryAllowed(entry: InflightEntry): boolean { + if (this.retryConfig.appendRetryPolicy === "all") { + return true; + } + return !!entry.args?.match_seq_num || !!entry.args?.fencing_token; + } + + private async enqueueBatch( + records: AppendRecord[], + args?: Omit & { precalculatedSize?: number }, + ): Promise<{ + ackPromise: Promise; + enqueuedPromise: Promise; + }> { + if (this.closing || this.pumpStopped) { + throw new S2Error({ message: "AppendSession is closed", status: 400 }); + } + if (this.pumpError) { + throw this.pumpError; + } + + const clonedRecords = this.cloneRecords(records); + const expectedCount = clonedRecords.length; + const meteredBytes = + args?.precalculatedSize ?? + clonedRecords.reduce((sum, record) => sum + meteredSizeBytes(record), 0); + + let resolveAck!: (ack: AppendAck) => void; + let rejectAck!: (err: S2Error) => void; + const ackPromise = new Promise((resolve, reject) => { + resolveAck = resolve; + rejectAck = reject; + }); + + let resolveEnqueued!: () => void; + let rejectEnqueued!: (err: S2Error) => void; + const enqueuedPromise = new Promise((resolve, reject) => { + resolveEnqueued = resolve; + rejectEnqueued = reject; + }); + + try { + await this.waitForCapacity(meteredBytes); + } catch (error) { + const err = s2Error(error); + rejectAck(err); + rejectEnqueued(err); + throw err; + } + + const entry: InflightEntry = { + records: clonedRecords, + args, + expectedCount, + meteredBytes, + enqueuedAt: Date.now(), + sent: false, + resolve: (ack) => resolveAck(ack), + reject: (err) => rejectAck(err), + }; + + this.notificationQueue.push({ type: "batch", entry }); + this.ensurePump(); + resolveEnqueued(); + return { ackPromise, enqueuedPromise }; + } + + async submit( + records: AppendRecord | AppendRecord[], + args?: Omit & { precalculatedSize?: number }, + ): Promise { + const batch = Array.isArray(records) ? records : [records]; + const { ackPromise } = await this.enqueueBatch(batch, args); + return ackPromise; + } + + acks(): AcksStream { + return this.readable as AcksStream; + } + + lastAckedPosition(): AppendAck | undefined { + return this._lastAckedPosition; + } + + async close(): Promise { + if (this.pumpStopped) { + return; + } + if (this.closePromise) { + return this.closePromise; + } + this.closing = true; + this.closePromise = new Promise((resolve, reject) => { + this.closeResolve = resolve; + this.closeReject = reject; + }); + this.ensurePump(); + this.notificationQueue.push({ type: "close" }); + return this.closePromise; + } + + async abort(reason?: unknown): Promise { + const error = + reason instanceof S2Error + ? reason + : new S2Error({ + message: reason?.toString() ?? "AppendSession aborted", + status: 499, + }); + this.closed = true; + this.notificationQueue.push({ type: "abort", error }); + this.ensurePump(); + } + + async [Symbol.asyncDispose](): Promise { + await this.close(); + } +} diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 1760b08..972ef3f 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -1,3 +1,4 @@ +import { UnknownFieldHandler } from "@protobuf-ts/runtime"; import type { S2RequestOptions } from "../../../../common.js"; import { RangeNotSatisfiableError, S2Error } from "../../../../error.js"; import { @@ -15,6 +16,8 @@ import { meteredSizeBytes } from "../../../../utils.js"; import { decodeFromBase64 } from "../../../base64.js"; import { EventStream } from "../../../event-stream.js"; import * as Redacted from "../../../redacted.js"; +import type { AppendResult, CloseResult } from "../../../result.js"; +import { err, errClose, ok, okClose } from "../../../result.js"; import type { AppendArgs, AppendRecord, @@ -23,63 +26,122 @@ import type { ReadArgs, ReadBatch, ReadRecord, + ReadResult, ReadSession, SessionTransport, TransportConfig, + TransportReadSession, } from "../../types.js"; import { streamAppend } from "./shared.js"; -export class FetchReadSession< - Format extends "string" | "bytes" = "string", -> extends EventStream> { +import last = UnknownFieldHandler.last; + +import createDebug from "debug"; +import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; + +const debug = createDebug("s2:fetch"); + +export class FetchReadSession + extends ReadableStream> + implements TransportReadSession +{ static async create( client: Client, name: string, args?: ReadArgs, options?: S2RequestOptions, ) { - console.log("FetchReadSession.create", name, args); + debug("FetchReadSession.create stream=%s args=%o", name, args); const { as, ...queryParams } = args ?? {}; - const response = await read({ - client, - path: { - stream: name, - }, - headers: { - accept: "text/event-stream", - ...(as === "bytes" ? { "s2-format": "base64" } : {}), - }, - query: queryParams, - parseAs: "stream", - ...options, - }); - if (response.error) { - if ("message" in response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - }); - } else { - // special case for 416 - Range Not Satisfiable - throw new RangeNotSatisfiableError({ - status: response.response.status, + + try { + const response = await read({ + client, + path: { + stream: name, + }, + headers: { + accept: "text/event-stream", + ...(as === "bytes" ? { "s2-format": "base64" } : {}), + }, + query: queryParams, + parseAs: "stream", + ...options, + }); + if (response.error) { + // Convert error to S2Error and return error session + const error = + "message" in response.error + ? new S2Error({ + message: response.error.message, + code: response.error.code ?? undefined, + status: response.response.status, + }) + : new RangeNotSatisfiableError({ + status: response.response.status, + }); + return FetchReadSession.createErrorSession(error); + } + if (!response.response.body) { + const error = new S2Error({ + message: "No body in SSE response", }); + return FetchReadSession.createErrorSession(error); } + const format = (args?.as ?? "string") as Format; + return new FetchReadSession(response.response.body, format); + } catch (error) { + // Catch any thrown errors (network failures, DNS errors, etc.) + const s2Error = + error instanceof S2Error + ? error + : new S2Error({ + message: String(error), + status: 502, // Bad Gateway - network/fetch failure + }); + return FetchReadSession.createErrorSession(s2Error); } - if (!response.response.body) { - throw new S2Error({ - message: "No body in SSE response", - }); - } - const format = (args?.as ?? "string") as Format; - return new FetchReadSession(response.response.body, format); } - private _lastReadPosition: StreamPosition | undefined = undefined; + /** + * Create a session that immediately emits an error result and closes. + * Used when errors occur during session creation. + */ + private static createErrorSession( + error: S2Error, + ): FetchReadSession { + // Create a custom instance that extends ReadableStream and emits error immediately + const stream = new ReadableStream>({ + start(controller) { + controller.enqueue({ ok: false, error }); + controller.close(); + }, + }); + + // Copy methods from stream to create a proper FetchReadSession + const session = Object.assign( + Object.create(FetchReadSession.prototype), + stream, + ); + session._nextReadPosition = undefined; + session._lastObservedTail = undefined; + + return session as FetchReadSession; + } + + private _nextReadPosition: StreamPosition | undefined = undefined; + private _lastObservedTail: StreamPosition | undefined = undefined; private constructor(stream: ReadableStream, format: Format) { - super(stream, (msg) => { + // Track error from parser + let parserError: S2Error | null = null; + + // Track last ping time for timeout detection (20s without a ping = timeout) + let lastPingTimeMs = performance.now(); + const PING_TIMEOUT_MS = 20000; // 20 seconds + + // Create EventStream that parses SSE and yields records + const eventStream = new EventStream>(stream, (msg) => { // Parse SSE events according to the S2 protocol if (msg.event === "batch" && msg.data) { const rawBatch: GeneratedReadBatch = JSON.parse(msg.data); @@ -109,81 +171,157 @@ export class FetchReadSession< } })() as ReadBatch; if (batch.tail) { - this._lastReadPosition = batch.tail; + this._lastObservedTail = batch.tail; + } + let lastRecord = batch.records?.at(-1); + if (lastRecord) { + this._nextReadPosition = { + seq_num: lastRecord.seq_num + 1, + timestamp: lastRecord.timestamp, + }; } return { done: false, batch: true, value: batch.records ?? [] }; } if (msg.event === "error") { - // Handle error events - throw new S2Error({ message: msg.data ?? "Unknown error" }); + // Store error and signal end of stream + // SSE error events are server errors - treat as 503 (Service Unavailable) for retry logic + debug("parse event error"); + parserError = new S2Error({ + message: msg.data ?? "Unknown error", + status: 503, + }); + return { done: true }; } + lastPingTimeMs = performance.now(); // Skip ping events and other events return { done: false }; }); - } - public lastReadPosition() { - return this._lastReadPosition; - } -} + // Wrap the EventStream to convert records to ReadResult and check for errors + const reader = eventStream.getReader(); + let done = false; -class AcksStream extends ReadableStream implements AsyncDisposable { - constructor( - setController: ( - controller: ReadableStreamDefaultController, - ) => void, - ) { super({ - start: (controller) => { - setController(controller); + pull: async (controller) => { + if (done) { + controller.close(); + return; + } + + // Check for ping timeout before reading + const now = performance.now(); + const timeSinceLastPingMs = now - lastPingTimeMs; + if (timeSinceLastPingMs > PING_TIMEOUT_MS) { + const timeoutError = new S2Error({ + message: `No ping received for ${Math.floor(timeSinceLastPingMs / 1000)}s (timeout: ${PING_TIMEOUT_MS / 1000}s)`, + status: 408, // Request Timeout + code: "TIMEOUT", + }); + debug("ping timeout detected, elapsed=%dms", timeSinceLastPingMs); + controller.enqueue({ ok: false, error: timeoutError }); + done = true; + controller.close(); + return; + } + + try { + // Calculate remaining time until timeout + const remainingTimeMs = PING_TIMEOUT_MS - timeSinceLastPingMs; + + // Race reader.read() against timeout + // This ensures we don't wait forever if server stops sending events + const result = await Promise.race([ + reader.read(), + new Promise<{ done: true; value: undefined }>((_, reject) => + setTimeout(() => { + const elapsed = performance.now() - lastPingTimeMs; + reject( + new S2Error({ + message: `No ping received for ${Math.floor(elapsed / 1000)}s (timeout: ${PING_TIMEOUT_MS / 1000}s)`, + status: 408, + code: "TIMEOUT", + }), + ); + }, remainingTimeMs), + ), + ]); + + if (result.done) { + done = true; + // Check if stream ended due to error + if (parserError) { + controller.enqueue({ ok: false, error: parserError }); + } + controller.close(); + } else { + // Emit successful result + controller.enqueue({ ok: true, value: result.value }); + } + } catch (error) { + // Convert unexpected errors to S2Error and emit as error result + const s2Err = + error instanceof S2Error + ? error + : new S2Error({ message: String(error), status: 500 }); + controller.enqueue({ ok: false, error: s2Err }); + done = true; + controller.close(); + } + }, + cancel: async () => { + await eventStream.cancel(); }, }); } - async [Symbol.asyncDispose]() { - await this.cancel("disposed"); + public nextReadPosition(): StreamPosition | undefined { + return this._nextReadPosition; } - // Polyfill for older browsers - [Symbol.asyncIterator](): AsyncIterableIterator { + public lastObservedTail(): StreamPosition | undefined { + return this._lastObservedTail; + } + + // Implement AsyncIterable (for await...of support) + [Symbol.asyncIterator](): AsyncIterableIterator> { const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; if (typeof fn === "function") return fn.call(this); const reader = this.getReader(); return { next: async () => { const r = await reader.read(); - if (r.done) { - reader.releaseLock(); - return { done: true, value: undefined }; - } + if (r.done) return { done: true, value: undefined }; return { done: false, value: r.value }; }, - throw: async (e) => { - await reader.cancel(e); + return: async (value?: any) => { reader.releaseLock(); - return { done: true, value: undefined }; + return { done: true, value }; }, - return: async () => { - await reader.cancel("done"); + throw: async (e?: any) => { reader.releaseLock(); - return { done: true, value: undefined }; + throw e; }, [Symbol.asyncIterator]() { return this; }, }; } + + // Implement AsyncDisposable + async [Symbol.asyncDispose](): Promise { + await this.cancel(); + } } +// Removed AcksStream - transport sessions no longer expose streams + /** - * Session for appending records to a stream. - * Queues append requests and ensures only one is in-flight at a time. + * "Dumb" transport session for appending records via HTTP/1.1. + * Queues append requests and ensures only one is in-flight at a time (single-flight). + * No backpressure, no retry logic, no streams - just submit/close with value-encoded errors. */ -export class FetchAppendSession - implements ReadableWritablePair, AsyncDisposable -{ - private _lastAckedPosition: AppendAck | undefined = undefined; +export class FetchAppendSession { private queue: Array<{ records: AppendRecord[]; fencing_token?: string; @@ -191,27 +329,15 @@ export class FetchAppendSession meteredSize: number; }> = []; private pendingResolvers: Array<{ - resolve: (ack: AppendAck) => void; - reject: (error: any) => void; + resolve: (result: AppendResult) => void; }> = []; private inFlight = false; private readonly options?: S2RequestOptions; private readonly stream: string; - private acksController: - | ReadableStreamDefaultController - | undefined; - private _readable: AcksStream; - private _writable: WritableStream; private closed = false; private processingPromise: Promise | null = null; - private queuedBytes = 0; - private readonly maxQueuedBytes: number; - private waitingForCapacity: Array<() => void> = []; private readonly client: Client; - public readonly readable: ReadableStream; - public readonly writable: WritableStream; - static async create( stream: string, transportConfig: TransportConfig, @@ -234,123 +360,65 @@ export class FetchAppendSession ) { this.options = requestOptions; this.stream = stream; - this.maxQueuedBytes = sessionOptions?.maxQueuedBytes ?? 10 * 1024 * 1024; // 10 MiB default this.client = createClient( createConfig({ baseUrl: transportConfig.baseUrl, auth: () => Redacted.value(transportConfig.accessToken), + headers: transportConfig.basinName + ? { "s2-basin": transportConfig.basinName } + : {}, }), ); - // Create the readable stream for acks - this._readable = new AcksStream((controller) => { - this.acksController = controller; - }); - this.readable = this._readable; - - // Create the writable stream - let writableController: WritableStreamDefaultController; - this._writable = new WritableStream({ - start: (controller) => { - writableController = controller; - }, - write: async (chunk) => { - // Calculate batch size - let batchMeteredSize = 0; - for (const record of chunk.records) { - batchMeteredSize += meteredSizeBytes(record as AppendRecord); - } - - // Wait for capacity if needed - while ( - this.queuedBytes + batchMeteredSize > this.maxQueuedBytes && - !this.closed - ) { - await new Promise((resolve) => { - this.waitingForCapacity.push(resolve); - }); - } - - // Submit the batch - this.submit( - chunk.records, - { - fencing_token: chunk.fencing_token ?? undefined, - match_seq_num: chunk.match_seq_num ?? undefined, - }, - batchMeteredSize, - ); - }, - close: async () => { - this.closed = true; - await this.waitForDrain(); - }, - abort: async (reason) => { - this.closed = true; - this.queue = []; - this.queuedBytes = 0; - - // Reject all pending promises - const error = new S2Error({ - message: `AppendSession was aborted: ${reason}`, - }); - for (const resolver of this.pendingResolvers) { - resolver.reject(error); - } - this.pendingResolvers = []; - - // Reject all waiting for capacity - for (const resolver of this.waitingForCapacity) { - resolver(); - } - this.waitingForCapacity = []; - }, - }); - this.writable = this._writable; - } - - async [Symbol.asyncDispose]() { - await this.close(); - } - - /** - * Get a stream of acknowledgements for appends. - */ - acks(): AcksStream { - return this._readable; } /** * Close the append session. * Waits for all pending appends to complete before resolving. + * Never throws - returns CloseResult. */ - async close(): Promise { - await this.writable.close(); + async close(): Promise { + try { + this.closed = true; + await this.waitForDrain(); + return okClose(); + } catch (error) { + const s2Err = + error instanceof S2Error + ? error + : new S2Error({ message: String(error), status: 500 }); + return errClose(s2Err); + } } /** * Submit an append request to the session. * The request will be queued and sent when no other request is in-flight. - * Returns a promise that resolves when the append is acknowledged or rejects on error. + * Never throws - returns AppendResult discriminated union. */ submit( records: AppendRecord | AppendRecord[], args?: { fencing_token?: string; match_seq_num?: number }, precalculatedSize?: number, - ): Promise { + ): Promise { + // Validate closed state if (this.closed) { - return Promise.reject( - new S2Error({ message: "AppendSession is closed" }), + return Promise.resolve( + err(new S2Error({ message: "AppendSession is closed", status: 400 })), ); } const recordsArray = Array.isArray(records) ? records : [records]; - // Validate batch size limits + // Validate batch size limits (non-retryable 400-level error) if (recordsArray.length > 1000) { - return Promise.reject( - new S2Error({ - message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`, - }), + return Promise.resolve( + err( + new S2Error({ + message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`, + status: 400, + code: "INVALID_ARGUMENT", + }), + ), ); } @@ -363,32 +431,37 @@ export class FetchAppendSession } if (batchMeteredSize > 1024 * 1024) { - return Promise.reject( - new S2Error({ - message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`, - }), + return Promise.resolve( + err( + new S2Error({ + message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`, + status: 400, + code: "INVALID_ARGUMENT", + }), + ), ); } - return new Promise((resolve, reject) => { + return new Promise((resolve) => { this.queue.push({ records: recordsArray, fencing_token: args?.fencing_token, match_seq_num: args?.match_seq_num, meteredSize: batchMeteredSize, }); - this.queuedBytes += batchMeteredSize; - this.pendingResolvers.push({ resolve, reject }); + this.pendingResolvers.push({ resolve }); // Start processing if not already running if (!this.processingPromise) { - this.processingPromise = this.processLoop(); + // Attach a catch to avoid unhandled rejection warnings on hard failures + this.processingPromise = this.processLoop().catch(() => {}); } }); } /** * Main processing loop that sends queued requests one at a time. + * Single-flight: only one request in progress at a time. */ private async processLoop(): Promise { while (this.queue.length > 0) { @@ -407,48 +480,32 @@ export class FetchAppendSession }, this.options, ); - this._lastAckedPosition = ack; - - // Emit ack to the acks stream if it exists - if (this.acksController) { - this.acksController.enqueue(ack); - } - - // Resolve the promise for this request - resolver.resolve(ack); - // Release capacity and wake up waiting writers - this.queuedBytes -= args.meteredSize; - while (this.waitingForCapacity.length > 0) { - const waiter = this.waitingForCapacity.shift()!; - waiter(); - // Only wake one at a time - let them check capacity again - break; - } + // Resolve with success result + resolver.resolve(ok(ack)); } catch (error) { - this.inFlight = false; - this.processingPromise = null; + // Convert error to S2Error and resolve with error result + const s2Err = + error instanceof S2Error + ? error + : new S2Error({ message: String(error), status: 502 }); - // Reject the promise for this request - resolver.reject(error); + // Resolve this request with error + resolver.resolve(err(s2Err)); - // Reject all remaining pending promises + // Resolve all remaining pending promises with the same error + // (transport failure affects all queued requests) for (const pendingResolver of this.pendingResolvers) { - pendingResolver.reject(error); + pendingResolver.resolve(err(s2Err)); } this.pendingResolvers = []; - // Clear the queue and reset queued bytes + // Clear the queue this.queue = []; - this.queuedBytes = 0; - // Wake up all waiting writers (they'll see the closed state or retry) - for (const waiter of this.waitingForCapacity) { - waiter(); - } - this.waitingForCapacity = []; - - // Do not rethrow here to avoid unhandled rejection; callers already received rejection + this.inFlight = false; + this.processingPromise = null; + return; } this.inFlight = false; @@ -467,15 +524,6 @@ export class FetchAppendSession while (this.queue.length > 0 || this.inFlight) { await new Promise((resolve) => setTimeout(resolve, 10)); } - - // Close the acks stream if it exists - if (this.acksController) { - this.acksController.close(); - } - } - - lastAckedPosition() { - return this._lastAckedPosition; } } @@ -502,11 +550,21 @@ export class FetchTransport implements SessionTransport { sessionOptions?: AppendSessionOptions, requestOptions?: S2RequestOptions, ): Promise { - return FetchAppendSession.create( - stream, - this.transportConfig, - sessionOptions, - requestOptions, + const opts = { + ...sessionOptions, + maxInflightBatches: 1, + } as AppendSessionOptions; + return RetryAppendSession.create( + (o) => { + return FetchAppendSession.create( + stream, + this.transportConfig, + o, + requestOptions, + ); + }, + opts, + this.transportConfig.retry, ); } @@ -515,6 +573,12 @@ export class FetchTransport implements SessionTransport { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - return FetchReadSession.create(this.client, stream, args, options); + return RetryReadSession.create( + (a) => { + return FetchReadSession.create(this.client, stream, a, options); + }, + args, + this.transportConfig.retry, + ); } } diff --git a/src/lib/stream/transport/s2s/framing.ts b/src/lib/stream/transport/s2s/framing.ts index 9384c03..15c5e25 100644 --- a/src/lib/stream/transport/s2s/framing.ts +++ b/src/lib/stream/transport/s2s/framing.ts @@ -80,7 +80,7 @@ export function frameMessage(opts: { * Parser for reading s2s frames from a stream */ export class S2SFrameParser { - private buffer: Uint8Array = new Uint8Array(0); + buffer: Uint8Array = new Uint8Array(0); /** * Add data to the parser buffer diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 4214a06..87f8a6c 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -6,6 +6,7 @@ */ import * as http2 from "node:http2"; +import createDebug from "debug"; import type { S2RequestOptions } from "../../../../common.js"; import { type Client, @@ -22,6 +23,9 @@ import { import { S2Error } from "../../../../index.js"; import { meteredSizeBytes } from "../../../../utils.js"; import * as Redacted from "../../../redacted.js"; +import type { AppendResult, CloseResult } from "../../../result.js"; +import { err, errClose, ok, okClose } from "../../../result.js"; +import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; import type { AppendArgs, AppendRecord, @@ -29,12 +33,17 @@ import type { AppendSessionOptions, ReadArgs, ReadRecord, + ReadResult, ReadSession, SessionTransport, TransportConfig, + TransportReadSession, } from "../../types.js"; +import { FetchReadSession } from "../fetch/index.js"; import { frameMessage, S2SFrameParser } from "./framing.js"; +const debug = createDebug("s2:s2s"); + export function buildProtoAppendInput( records: AppendRecord[], args: AppendArgs, @@ -94,14 +103,20 @@ export class S2STransport implements SessionTransport { sessionOptions?: AppendSessionOptions, requestOptions?: S2RequestOptions, ): Promise { - return S2SAppendSession.create( - this.transportConfig.baseUrl, - this.transportConfig.accessToken, - stream, - () => this.getConnection(), - this.transportConfig.basinName, + return RetryAppendSession.create( + (opts) => { + return S2SAppendSession.create( + this.transportConfig.baseUrl, + this.transportConfig.accessToken, + stream, + () => this.getConnection(), + this.transportConfig.basinName, + opts, + requestOptions, + ); + }, sessionOptions, - requestOptions, + this.transportConfig.retry, ); } @@ -110,14 +125,29 @@ export class S2STransport implements SessionTransport { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - return S2SReadSession.create( - this.transportConfig.baseUrl, - this.transportConfig.accessToken, - stream, + // return S2SReadSession.create( + // this.transportConfig.baseUrl, + // this.transportConfig.accessToken, + // stream, + // args, + // options, + // () => this.getConnection(), + // this.transportConfig.basinName, + // ); + return RetryReadSession.create( + (a) => { + return S2SReadSession.create( + this.transportConfig.baseUrl, + this.transportConfig.accessToken, + stream, + a, + options, + () => this.getConnection(), + this.transportConfig.basinName, + ); + }, args, - options, - () => this.getConnection(), - this.transportConfig.basinName, + this.transportConfig.retry, ); } @@ -184,11 +214,13 @@ export class S2STransport implements SessionTransport { } class S2SReadSession - extends ReadableStream> - implements ReadSession + extends ReadableStream> + implements TransportReadSession { private http2Stream?: http2.ClientHttp2Stream; private _lastReadPosition?: StreamPosition; + private _nextReadPosition?: StreamPosition; + private _lastObservedTail?: StreamPosition; private parser = new S2SFrameParser(); static async create( @@ -227,6 +259,10 @@ class S2SReadSession let http2Stream: http2.ClientHttp2Stream | undefined; let lastReadPosition: StreamPosition | undefined; + // Track timeout for detecting when server stops sending data + const TAIL_TIMEOUT_MS = 20000; // 20 seconds + let timeoutTimer: NodeJS.Timeout | undefined; + super({ start: async (controller) => { let controllerClosed = false; @@ -234,6 +270,10 @@ class S2SReadSession const safeClose = () => { if (!controllerClosed) { controllerClosed = true; + if (timeoutTimer) { + clearTimeout(timeoutTimer); + timeoutTimer = undefined; + } try { controller.close(); } catch { @@ -244,11 +284,40 @@ class S2SReadSession const safeError = (err: unknown) => { if (!controllerClosed) { controllerClosed = true; - controller.error(err); + if (timeoutTimer) { + clearTimeout(timeoutTimer); + timeoutTimer = undefined; + } + // Convert error to S2Error and enqueue as error result + const s2Err = + err instanceof S2Error + ? err + : new S2Error({ message: String(err), status: 500 }); + controller.enqueue({ ok: false, error: s2Err }); + controller.close(); } }; + // Helper to start/reset the timeout timer + // Resets on every tail received, fires only if no tail for 20s + const resetTimeoutTimer = () => { + if (timeoutTimer) { + clearTimeout(timeoutTimer); + } + timeoutTimer = setTimeout(() => { + const timeoutError = new S2Error({ + message: `No tail received for ${TAIL_TIMEOUT_MS / 1000}s`, + status: 408, // Request Timeout + code: "TIMEOUT", + }); + debug("tail timeout detected"); + safeError(timeoutError); + }, TAIL_TIMEOUT_MS); + }; + try { + // Start the timeout timer - will fire in 20s if no tail received + resetTimeoutTimer(); const connection = await getConnection(); // Build query string @@ -299,96 +368,147 @@ class S2SReadSession responseCode = headers[":status"] ?? 500; }); + connection.on("goaway", (errorCode, lastStreamID, opaqueData) => { + debug("received GOAWAY from server"); + }); + + stream.on("error", (err) => { + safeError(err); + }); + stream.on("data", (chunk: Buffer) => { - if ((responseCode ?? 500) >= 400) { - const errorText = textDecoder.decode(chunk); - try { - const errorJson = JSON.parse(errorText); - safeError( - new S2Error({ - message: errorJson.message ?? "Unknown error", - code: errorJson.code, - status: responseCode, - }), - ); - } catch { - safeError( - new S2Error({ - message: errorText || "Unknown error", - status: responseCode, - }), - ); + try { + if ((responseCode ?? 500) >= 400) { + const errorText = textDecoder.decode(chunk); + try { + const errorJson = JSON.parse(errorText); + safeError( + new S2Error({ + message: errorJson.message ?? "Unknown error", + code: errorJson.code, + status: responseCode, + }), + ); + } catch { + safeError( + new S2Error({ + message: errorText || "Unknown error", + status: responseCode, + }), + ); + } + return; } - } - // Buffer already extends Uint8Array in Node.js, no need to convert - parser.push(chunk); - - let frame = parser.parseFrame(); - while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); + // Buffer already extends Uint8Array in Node.js, no need to convert + parser.push(chunk); + + let frame = parser.parseFrame(); + while (frame) { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + try { + const errorJson = JSON.parse(errorText); + safeError( + new S2Error({ + message: errorJson.message ?? "Unknown error", + code: errorJson.code, + status: frame.statusCode, + }), + ); + } catch { + safeError( + new S2Error({ + message: errorText || "Unknown error", + status: frame.statusCode, + }), + ); + } + } else { + safeClose(); + } + stream.close(); + } else { + // Parse ReadBatch try { - const errorJson = JSON.parse(errorText); - safeError( - new S2Error({ - message: errorJson.message ?? "Unknown error", - code: errorJson.code, - status: frame.statusCode, - }), - ); - } catch { + const protoBatch = ProtoReadBatch.fromBinary(frame.body); + + resetTimeoutTimer(); + + // Update tail from batch + if (protoBatch.tail) { + const tail = convertStreamPosition(protoBatch.tail); + lastReadPosition = tail; + this._lastReadPosition = tail; + this._lastObservedTail = tail; + debug("received tail"); + } + + // Enqueue each record and track next position + for (const record of protoBatch.records) { + const converted = this.convertRecord( + record, + as ?? ("string" as Format), + textDecoder, + ); + controller.enqueue({ ok: true, value: converted }); + + // Update next read position to after this record + if (record.seqNum !== undefined) { + this._nextReadPosition = { + seq_num: Number(record.seqNum) + 1, + timestamp: 0, + }; + } + } + } catch (err) { safeError( new S2Error({ - message: errorText || "Unknown error", - status: frame.statusCode, + message: `Failed to parse ReadBatch: ${err}`, }), ); } - } else { - safeClose(); } - stream.close(); - } else { - // Parse ReadBatch - try { - const protoBatch = ProtoReadBatch.fromBinary(frame.body); - - // Update position from tail - if (protoBatch.tail) { - lastReadPosition = convertStreamPosition(protoBatch.tail); - // Assign to instance property - this._lastReadPosition = lastReadPosition; - } - // Enqueue each record - for (const record of protoBatch.records) { - const converted = this.convertRecord( - record, - as ?? ("string" as Format), - textDecoder, - ); - controller.enqueue(converted); - } - } catch (err) { - safeError( - new S2Error({ - message: `Failed to parse ReadBatch: ${err}`, - }), - ); - } + frame = parser.parseFrame(); } - - frame = parser.parseFrame(); + } catch (error) { + safeError( + error instanceof S2Error + ? error + : new S2Error({ + message: `Failed to process read data: ${error}`, + status: 500, + }), + ); } }); - stream.on("error", (err) => { - safeError(err); + stream.on("end", () => { + if (stream.rstCode != 0) { + debug!("stream reset code=%d", stream.rstCode); + safeError( + new S2Error({ + message: `Stream ended with error: ${stream.rstCode}`, + status: 500, + code: "stream reset", + }), + ); + } }); stream.on("close", () => { - safeClose(); + if (parser.hasData()) { + safeError( + new S2Error({ + message: "Stream closed with unparsed data remaining", + status: 500, + code: "STREAM_CLOSED_PREMATURELY", + }), + ); + } else { + safeClose(); + } }); } catch (err) { safeError(err); @@ -454,7 +574,7 @@ class S2SReadSession } // Polyfill for older browsers / Node.js environments - [Symbol.asyncIterator](): AsyncIterableIterator> { + [Symbol.asyncIterator](): AsyncIterableIterator> { const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; if (typeof fn === "function") return fn.call(this); const reader = this.getReader(); @@ -483,92 +603,35 @@ class S2SReadSession }; } - lastReadPosition(): StreamPosition | undefined { - return this._lastReadPosition; + nextReadPosition(): StreamPosition | undefined { + return this._nextReadPosition; + } + + lastObservedTail(): StreamPosition | undefined { + return this._lastObservedTail; } } /** * AcksStream for S2S append session */ -class S2SAcksStream - extends ReadableStream - implements AsyncDisposable -{ - constructor( - setController: ( - controller: ReadableStreamDefaultController, - ) => void, - ) { - super({ - start: (controller) => { - setController(controller); - }, - }); - } - - async [Symbol.asyncDispose]() { - await this.cancel("disposed"); - } - - // Polyfill for older browsers - [Symbol.asyncIterator](): AsyncIterableIterator { - const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; - if (typeof fn === "function") return fn.call(this); - const reader = this.getReader(); - return { - next: async () => { - const r = await reader.read(); - if (r.done) { - reader.releaseLock(); - return { done: true, value: undefined }; - } - return { done: false, value: r.value }; - }, - throw: async (e) => { - await reader.cancel(e); - reader.releaseLock(); - return { done: true, value: undefined }; - }, - return: async () => { - await reader.cancel("done"); - reader.releaseLock(); - return { done: true, value: undefined }; - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - } -} +// Removed S2SAcksStream - transport sessions no longer expose streams /** - * S2S Append Session for pipelined writes - * Unlike fetch-based append, writes don't block on acks - only on submission + * "Dumb" transport session for appending records via HTTP/2. + * Pipelined: multiple requests can be in-flight simultaneously. + * No backpressure, no retry logic, no streams - just submit/close with value-encoded errors. */ -class S2SAppendSession - implements ReadableWritablePair, AsyncDisposable -{ +class S2SAppendSession { private http2Stream?: http2.ClientHttp2Stream; - private _lastAckedPosition?: AppendAck; private parser = new S2SFrameParser(); - private acksController?: ReadableStreamDefaultController; - private _readable: S2SAcksStream; - private _writable: WritableStream; private closed = false; - private queuedBytes = 0; - private readonly maxQueuedBytes: number; - private waitingForCapacity: Array<() => void> = []; private pendingAcks: Array<{ - resolve: (ack: AppendAck) => void; - reject: (error: any) => void; + resolve: (result: AppendResult) => void; batchSize: number; }> = []; private initPromise?: Promise; - public readonly readable: ReadableStream; - public readonly writable: WritableStream; - static async create( baseUrl: string, bearerToken: Redacted.Redacted, @@ -598,95 +661,8 @@ class S2SAppendSession sessionOptions?: AppendSessionOptions, private options?: S2RequestOptions, ) { - this.maxQueuedBytes = sessionOptions?.maxQueuedBytes ?? 10 * 1024 * 1024; // 10 MiB default - - // Create the readable stream for acks - this._readable = new S2SAcksStream((controller) => { - this.acksController = controller; - }); - this.readable = this._readable; - - // Create the writable stream - this._writable = new WritableStream({ - start: async (controller) => { - this.initPromise = this.initializeStream(); - await this.initPromise; - }, - write: async (chunk) => { - if (this.closed) { - throw new S2Error({ message: "AppendSession is closed" }); - } - - const recordsArray = Array.isArray(chunk.records) - ? chunk.records - : [chunk.records]; - - // Validate batch size limits - if (recordsArray.length > 1000) { - throw new S2Error({ - message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`, - }); - } - - // Calculate metered size - let batchMeteredSize = 0; - for (const record of recordsArray) { - batchMeteredSize += meteredSizeBytes(record); - } - - if (batchMeteredSize > 1024 * 1024) { - throw new S2Error({ - message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`, - }); - } - - // Wait for capacity if needed (backpressure) - while ( - this.queuedBytes + batchMeteredSize > this.maxQueuedBytes && - !this.closed - ) { - await new Promise((resolve) => { - this.waitingForCapacity.push(resolve); - }); - } - - if (this.closed) { - throw new S2Error({ message: "AppendSession is closed" }); - } - - // Send the batch immediately (pipelined) - // Returns when frame is sent, not when ack is received - await this.sendBatchNonBlocking(recordsArray, chunk, batchMeteredSize); - }, - close: async () => { - this.closed = true; - await this.closeStream(); - }, - abort: async (reason) => { - this.closed = true; - this.queuedBytes = 0; - - // Reject all pending acks - const error = new S2Error({ - message: `AppendSession was aborted: ${reason}`, - }); - for (const pending of this.pendingAcks) { - pending.reject(error); - } - this.pendingAcks = []; - - // Wake up all waiting for capacity - for (const resolver of this.waitingForCapacity) { - resolver(); - } - this.waitingForCapacity = []; - - if (this.http2Stream && !this.http2Stream.closed) { - this.http2Stream.close(); - } - }, - }); - this.writable = this._writable; + // No stream setup - transport is "dumb" + // Initialization happens lazily on first submit } private async initializeStream(): Promise { @@ -715,184 +691,114 @@ class S2SAppendSession }); const textDecoder = new TextDecoder(); - let controllerClosed = false; - const safeClose = () => { - if (!controllerClosed && this.acksController) { - controllerClosed = true; - try { - this.acksController.close(); - } catch { - // Controller may already be closed, ignore - } - } - }; + const safeError = (error: unknown) => { + const s2Err = + error instanceof S2Error + ? error + : new S2Error({ message: String(error), status: 502 }); - const safeError = (err: unknown) => { - if (!controllerClosed && this.acksController) { - controllerClosed = true; - this.acksController.error(err); - } - - // Reject all pending acks + // Resolve all pending acks with error result for (const pending of this.pendingAcks) { - pending.reject(err); + pending.resolve(err(s2Err)); } this.pendingAcks = []; }; // Handle incoming data (acks) stream.on("data", (chunk: Buffer) => { - this.parser.push(chunk); - - let frame = this.parser.parseFrame(); - while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); - try { - const errorJson = JSON.parse(errorText); - safeError( - new S2Error({ - message: errorJson.message ?? "Unknown error", - code: errorJson.code, - status: frame.statusCode, - }), - ); - } catch { - safeError( - new S2Error({ - message: errorText || "Unknown error", - status: frame.statusCode, - }), - ); + try { + this.parser.push(chunk); + + let frame = this.parser.parseFrame(); + while (frame) { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + const status = frame.statusCode ?? 500; + try { + const errorJson = JSON.parse(errorText); + const message = errorJson.message ?? "Unknown error"; + const code = errorJson.code; + queueMicrotask(() => + safeError(new S2Error({ message, code, status })), + ); + } catch { + const message = errorText || "Unknown error"; + queueMicrotask(() => + safeError(new S2Error({ message, status })), + ); + } } + stream.close(); } else { - safeClose(); - } - stream.close(); - } else { - // Parse AppendAck - try { - const protoAck = ProtoAppendAck.fromBinary(frame.body); - - const ack = convertAppendAck(protoAck); - - this._lastAckedPosition = ack; - - // Enqueue to readable stream - if (this.acksController) { - this.acksController.enqueue(ack); - } - - // Resolve the pending ack promise - const pending = this.pendingAcks.shift(); - if (pending) { - pending.resolve(ack); - - // Release capacity - this.queuedBytes -= pending.batchSize; + // Parse AppendAck + try { + const protoAck = ProtoAppendAck.fromBinary(frame.body); + const ack = convertAppendAck(protoAck); - // Wake up one waiting writer - if (this.waitingForCapacity.length > 0) { - const waiter = this.waitingForCapacity.shift()!; - waiter(); + // Resolve the pending ack promise (FIFO) + const pending = this.pendingAcks.shift(); + if (pending) { + pending.resolve(ok(ack)); } + } catch (parseErr) { + queueMicrotask(() => + safeError( + new S2Error({ + message: `Failed to parse AppendAck: ${parseErr}`, + status: 500, + }), + ), + ); } - } catch (err) { - safeError( - new S2Error({ - message: `Failed to parse AppendAck: ${err}`, - }), - ); } - } - frame = this.parser.parseFrame(); + frame = this.parser.parseFrame(); + } + } catch (error) { + queueMicrotask(() => safeError(error)); } }); - stream.on("error", (err: Error) => { - safeError(err); + stream.on("error", (streamErr: Error) => { + queueMicrotask(() => safeError(streamErr)); }); stream.on("close", () => { - safeClose(); - }); - } - - /** - * Send a batch non-blocking (returns when frame is sent, not when ack is received) - */ - private sendBatchNonBlocking( - records: AppendRecord[], - args: AppendArgs, - batchMeteredSize: number, - ): Promise { - if (!this.http2Stream || this.http2Stream.closed) { - return Promise.reject( - new S2Error({ message: "HTTP/2 stream is not open" }), - ); - } - - // Convert to protobuf AppendInput - const protoInput = buildProtoAppendInput(records, args); - - const bodyBytes = ProtoAppendInput.toBinary(protoInput); - - // Frame the message - const frame = frameMessage({ - terminal: false, - body: bodyBytes, - }); - - // This promise resolves when the frame is written (not when ack is received) - return new Promise((resolve, reject) => { - // Track pending ack - will be resolved when ack arrives - const ackPromise = { - resolve: () => {}, - reject, - batchSize: batchMeteredSize, - }; - this.pendingAcks.push(ackPromise); - - this.queuedBytes += batchMeteredSize; - - // Send the frame (pipelined) - this.http2Stream!.write(frame, (err) => { - if (err) { - // Remove from pending acks on write error - const idx = this.pendingAcks.indexOf(ackPromise); - if (idx !== -1) { - this.pendingAcks.splice(idx, 1); - this.queuedBytes -= batchMeteredSize; - } - reject(err); - } else { - // Frame written successfully - resolve immediately (pipelined) - resolve(); - } - }); + // Stream closed - resolve any remaining pending acks with error + // This can happen if the server closes the stream without sending all acks + if (this.pendingAcks.length > 0) { + queueMicrotask(() => + safeError( + new S2Error({ + message: "Stream closed with pending acks", + status: 502, + code: "BAD_GATEWAY", + }), + ), + ); + } }); } /** - * Send a batch and wait for ack (used by submit method) + * Send a batch and wait for ack. Returns AppendResult (never throws). + * Pipelined: multiple sends can be in-flight; acks resolve FIFO. */ private sendBatch( records: AppendRecord[], args: AppendArgs, batchMeteredSize: number, - ): Promise { + ): Promise { if (!this.http2Stream || this.http2Stream.closed) { - return Promise.reject( - new S2Error({ message: "HTTP/2 stream is not open" }), + return Promise.resolve( + err(new S2Error({ message: "HTTP/2 stream is not open", status: 502 })), ); } // Convert to protobuf AppendInput const protoInput = buildProtoAppendInput(records, args); - const bodyBytes = ProtoAppendInput.toBinary(protoInput); // Frame the message @@ -901,89 +807,102 @@ class S2SAppendSession body: bodyBytes, }); - // Track pending ack - this promise resolves when the ack is received - return new Promise((resolve, reject) => { + // Track pending ack - this promise resolves when the ack is received (FIFO) + return new Promise((resolve) => { this.pendingAcks.push({ resolve, - reject, batchSize: batchMeteredSize, }); - this.queuedBytes += batchMeteredSize; - - // Send the frame (non-blocking - pipelined) - this.http2Stream!.write(frame, (err) => { - if (err) { + // Send the frame (pipelined - non-blocking) + this.http2Stream!.write(frame, (writeErr) => { + if (writeErr) { // Remove from pending acks on write error - const idx = this.pendingAcks.findIndex((p) => p.reject === reject); + const idx = this.pendingAcks.findIndex((p) => p.resolve === resolve); if (idx !== -1) { this.pendingAcks.splice(idx, 1); - this.queuedBytes -= batchMeteredSize; } - reject(err); + // Resolve with error result + const s2Err = + writeErr instanceof S2Error + ? writeErr + : new S2Error({ message: String(writeErr), status: 502 }); + resolve(err(s2Err)); } - // Write completed, but promise resolves when ack is received + // Write completed successfully - promise resolves later when ack is received }); }); } - private async closeStream(): Promise { - // Wait for all pending acks - while (this.pendingAcks.length > 0) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - - // Close the HTTP/2 stream (client doesn't send terminal frame for clean close) - if (this.http2Stream && !this.http2Stream.closed) { - this.http2Stream.end(); - } - } - - async [Symbol.asyncDispose]() { - await this.close(); - } - - /** - * Get a stream of acknowledgements for appends. - */ - acks(): S2SAcksStream { - return this._readable; - } - /** * Close the append session. * Waits for all pending appends to complete before resolving. + * Never throws - returns CloseResult. */ - async close(): Promise { - await this.writable.close(); + async close(): Promise { + try { + this.closed = true; + + // Wait for all pending acks to complete + while (this.pendingAcks.length > 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + // Close the HTTP/2 stream (client doesn't send terminal frame for clean close) + if (this.http2Stream && !this.http2Stream.closed) { + this.http2Stream.end(); + } + + return okClose(); + } catch (error) { + const s2Err = + error instanceof S2Error + ? error + : new S2Error({ message: String(error), status: 500 }); + return errClose(s2Err); + } } /** * Submit an append request to the session. - * Returns a promise that resolves with the ack when received. + * Returns AppendResult (never throws). + * Pipelined: multiple submits can be in-flight; acks resolve FIFO. */ async submit( records: AppendRecord | AppendRecord[], args?: { fencing_token?: string; match_seq_num?: number }, - ): Promise { + ): Promise { + // Validate closed state if (this.closed) { - return Promise.reject( - new S2Error({ message: "AppendSession is closed" }), + return err( + new S2Error({ message: "AppendSession is closed", status: 400 }), ); } - // Wait for initialization - if (this.initPromise) { + // Lazy initialize HTTP/2 stream on first submit + if (!this.initPromise) { + this.initPromise = this.initializeStream(); + } + + try { await this.initPromise; + } catch (initErr) { + const s2Err = + initErr instanceof S2Error + ? initErr + : new S2Error({ message: String(initErr), status: 502 }); + return err(s2Err); } const recordsArray = Array.isArray(records) ? records : [records]; - // Validate batch size limits + // Validate batch size limits (non-retryable 400-level error) if (recordsArray.length > 1000) { - return Promise.reject( + return err( new S2Error({ message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`, + status: 400, + code: "INVALID_ARGUMENT", }), ); } @@ -995,9 +914,11 @@ class S2SAppendSession } if (batchMeteredSize > 1024 * 1024) { - return Promise.reject( + return err( new S2Error({ message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`, + status: 400, + code: "INVALID_ARGUMENT", }), ); } @@ -1012,10 +933,6 @@ class S2SAppendSession batchMeteredSize, ); } - - lastAckedPosition(): AppendAck | undefined { - return this._lastAckedPosition; - } } /** diff --git a/src/lib/stream/types.ts b/src/lib/stream/types.ts index 5b4e9fb..b9c6748 100644 --- a/src/lib/stream/types.ts +++ b/src/lib/stream/types.ts @@ -1,4 +1,5 @@ -import type { S2RequestOptions } from "../../common.js"; +import type { RetryConfig, S2RequestOptions } from "../../common.js"; +import type { S2Error } from "../../error.js"; import type { AppendAck, AppendInput as GeneratedAppendInput, @@ -62,6 +63,23 @@ export interface AcksStream extends ReadableStream, AsyncIterable {} +/** + * Transport-facing interface for "dumb" append sessions. + * Transports only implement submit/close with value-encoded errors (discriminated unions). + * No backpressure, no retry, no streams - RetryAppendSession adds those. + */ +export interface TransportAppendSession { + submit( + records: AppendRecord | AppendRecord[], + args?: Omit & { precalculatedSize?: number }, + ): Promise; + close(): Promise; +} + +/** + * Public AppendSession interface with retry, backpressure, and streams. + * This is what users interact with - implemented by RetryAppendSession. + */ export interface AppendSession extends ReadableWritablePair, AsyncDisposable { @@ -72,18 +90,60 @@ export interface AppendSession acks(): AcksStream; close(): Promise; lastAckedPosition(): AppendAck | undefined; + /** + * If the session has failed, returns the original fatal error that caused + * the pump to stop. Returns undefined when the session has not failed. + */ + failureCause(): S2Error | undefined; } +/** + * Result type for transport-level read operations. + * Transport sessions yield ReadResult instead of throwing errors. + */ +export type ReadResult = + | { ok: true; value: ReadRecord } + | { ok: false; error: S2Error }; + +/** + * Transport-level read session interface. + * Transport implementations yield ReadResult and never throw errors from the stream. + * RetryReadSession wraps these and converts them to the public ReadSession interface. + */ +export interface TransportReadSession< + Format extends "string" | "bytes" = "string", +> extends ReadableStream>, + AsyncIterable>, + AsyncDisposable { + nextReadPosition(): StreamPosition | undefined; + lastObservedTail(): StreamPosition | undefined; +} + +/** + * Public-facing read session interface. + * Yields records directly and propagates errors by throwing (standard stream behavior). + */ export interface ReadSession extends ReadableStream>, AsyncIterable>, AsyncDisposable { - lastReadPosition(): StreamPosition | undefined; + nextReadPosition(): StreamPosition | undefined; + lastObservedTail(): StreamPosition | undefined; } export interface AppendSessionOptions { - /** Maximum bytes to queue before applying backpressure (default: 10 MiB) */ + /** + * Maximum bytes to queue before applying backpressure (default: 10 MiB). + * Enforced by RetryAppendSession; underlying transports do not apply + * byte-based backpressure on their own. + */ maxQueuedBytes?: number; + /** + * Maximum number of batches allowed in-flight (including queued) before + * applying backpressure. This is enforced by RetryAppendSession; underlying + * transport sessions do not implement their own backpressure. + */ + maxInflightBatches?: number; } export interface SessionTransport { @@ -109,4 +169,8 @@ export interface TransportConfig { * Basin name to include in s2-basin header when using account endpoint */ basinName?: string; + /** + * Retry configuration inherited from the top-level client + */ + retry?: RetryConfig; } diff --git a/src/stream.ts b/src/stream.ts index ea2630d..5a41127 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,3 +1,4 @@ +import createDebug from "debug"; import type { RetryConfig, S2RequestOptions } from "./common.js"; import { S2Error, withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; @@ -19,7 +20,6 @@ import type { SessionTransport, TransportConfig, } from "./lib/stream/types.js"; -import createDebug from "debug"; const debug = createDebug("s2:stream"); @@ -123,7 +123,11 @@ export class S2Stream { }, (config, error) => { if ((config.appendRetryPolicy ?? "noSideEffects") === "noSideEffects") { - return !!args?.match_seq_num; + return ( + !!args?.match_seq_num || + error.status === 429 || + error.status === 502 + ); } else { return true; } diff --git a/src/tests/appendSession.test.ts b/src/tests/appendSession.test.ts index a8fb73c..313aed2 100644 --- a/src/tests/appendSession.test.ts +++ b/src/tests/appendSession.test.ts @@ -7,11 +7,12 @@ import { S2Stream } from "../stream.js"; // Minimal Client shape to satisfy S2Stream constructor; we won't use it directly const fakeClient: any = {}; -const makeStream = () => +const makeStream = (retry?: { maxAttempts?: number }) => new S2Stream("test-stream", fakeClient, { baseUrl: "https://test.b.aws.s2.dev", accessToken: Redacted.make("test-access-token"), forceTransport: "fetch", + retry, }); const makeAck = (n: number): AppendAck => ({ @@ -81,9 +82,14 @@ describe("AppendSession", () => { } })(); - await session.submit([{ body: "a" }]); - await session.submit([{ body: "b" }]); + const ack1 = await session.submit([{ body: "a" }]); + const ack2 = await session.submit([{ body: "b" }]); + // Verify acks were received before closing + expect(ack1).toBeTruthy(); + expect(ack2).toBeTruthy(); + + // Close session - with interruptible sleep, pump will wake immediately await session.close(); await consumer; @@ -103,6 +109,8 @@ describe("AppendSession", () => { const p2 = session.submit([{ body: "y" }]); await Promise.all([p1, p2]); + + // Close - with interruptible sleep, pump will wake immediately await session.close(); await expect(p1).resolves.toBeTruthy(); @@ -123,9 +131,12 @@ describe("AppendSession", () => { }); it("error during processing rejects current and queued, clears queue", async () => { - const stream = makeStream(); + // Create stream with no retries to test immediate failure + const stream = makeStream({ maxAttempts: 0 }); - streamAppendSpy.mockRejectedValueOnce(new Error("boom")); + // With retry enabled, the first error will trigger recovery and retry + // So we need to mock multiple failures to exhaust retries + streamAppendSpy.mockRejectedValue(new Error("boom")); const session = await stream.appendSession(); @@ -135,14 +146,15 @@ describe("AppendSession", () => { p1.catch(() => {}); p2.catch(() => {}); + // Advance timers to allow pump to attempt processing + await vi.advanceTimersByTimeAsync(10); + await expect(p1).rejects.toBeTruthy(); await expect(p2).rejects.toBeTruthy(); - // After error, queue should be empty; new submit should restart processing - streamAppendSpy.mockResolvedValueOnce(makeAck(3)); + // After fatal error, session is dead - new submits should also reject const p3 = session.submit([{ body: "c" }]); - await expect(p3).resolves.toBeTruthy(); - expect(streamAppendSpy).toHaveBeenCalledTimes(2); // 1 throw + 1 success + await expect(p3).rejects.toBeTruthy(); }); it("updates lastSeenPosition after successful append", async () => { @@ -210,6 +222,7 @@ describe("AppendSession", () => { expect(thirdWriteStarted).toBe(true); expect(streamAppendSpy).toHaveBeenCalledTimes(3); + // Close - with interruptible sleep, pump will wake immediately await writer.close(); }); }); diff --git a/src/tests/batcher-session.test.ts b/src/tests/batcher-session.test.ts index e7eb817..fdf7ad6 100644 --- a/src/tests/batcher-session.test.ts +++ b/src/tests/batcher-session.test.ts @@ -35,7 +35,17 @@ describe("BatchTransform + AppendSession integration", () => { it("linger-driven batching yields single session submission", async () => { const stream = makeStream(); const session = await stream.appendSession(); - streamAppendSpy.mockResolvedValue(makeAck(1)); + // Mock returns ack based on number of records submitted + let cumulativeSeq = 0; + streamAppendSpy.mockImplementation((_0: any, _1: any, records: any[]) => { + const start = cumulativeSeq; + cumulativeSeq += records.length; + return Promise.resolve({ + start: { seq_num: start, timestamp: 0 }, + end: { seq_num: cumulativeSeq, timestamp: 0 }, + tail: { seq_num: cumulativeSeq, timestamp: 0 }, + }); + }); const batcher = new BatchTransform({ lingerDurationMillis: 10, @@ -61,8 +71,17 @@ describe("BatchTransform + AppendSession integration", () => { it("batch overflow increments match_seq_num across multiple flushes", async () => { const stream = makeStream(); const session = await stream.appendSession(); - streamAppendSpy.mockResolvedValueOnce(makeAck(1)); - streamAppendSpy.mockResolvedValueOnce(makeAck(2)); + // Mock returns ack based on number of records submitted + let cumulativeSeq = 0; + streamAppendSpy.mockImplementation((_0: any, _1: any, records: any[]) => { + const start = cumulativeSeq; + cumulativeSeq += records.length; + return Promise.resolve({ + start: { seq_num: start, timestamp: 0 }, + end: { seq_num: cumulativeSeq, timestamp: 0 }, + tail: { seq_num: cumulativeSeq, timestamp: 0 }, + }); + }); const batcher = new BatchTransform({ lingerDurationMillis: 0, @@ -79,6 +98,8 @@ describe("BatchTransform + AppendSession integration", () => { await writer.write({ body: "3" }); await writer.close(); + // Advance timers to allow linger flushes to complete + await vi.advanceTimersByTimeAsync(10); await pipePromise; expect(streamAppendSpy).toHaveBeenCalledTimes(2); @@ -93,7 +114,17 @@ describe("BatchTransform + AppendSession integration", () => { it("batches are acknowledged via session.acks()", async () => { const stream = makeStream(); const session = await stream.appendSession(); - streamAppendSpy.mockResolvedValue(makeAck(123)); + // Mock returns ack based on number of records submitted + let cumulativeSeq = 0; + streamAppendSpy.mockImplementation((_0: any, _1: any, records: any[]) => { + const start = cumulativeSeq; + cumulativeSeq += records.length; + return Promise.resolve({ + start: { seq_num: start, timestamp: 0 }, + end: { seq_num: cumulativeSeq, timestamp: 0 }, + tail: { seq_num: cumulativeSeq, timestamp: 0 }, + }); + }); const batcher = new BatchTransform({ lingerDurationMillis: 0, @@ -121,6 +152,6 @@ describe("BatchTransform + AppendSession integration", () => { await acksPromise; expect(acks).toHaveLength(1); - expect(acks[0]?.end.seq_num).toBe(123); + expect(acks[0]?.end.seq_num).toBe(1); // 1 record written }); }); diff --git a/src/tests/readSession.e2e.test.ts b/src/tests/readSession.e2e.test.ts index dba3816..5ef4cae 100644 --- a/src/tests/readSession.e2e.test.ts +++ b/src/tests/readSession.e2e.test.ts @@ -96,14 +96,14 @@ describe("ReadSession Integration Tests", () => { }); // Initially streamPosition should be undefined - expect(session.lastReadPosition()).toBeUndefined(); + expect(session.nextReadPosition()).toBeUndefined(); const records: Array<{ seq_num: number }> = []; for await (const record of session) { records.push({ seq_num: record.seq_num }); // streamPosition should be updated after reading - if (session.lastReadPosition()) { - expect(session.lastReadPosition()?.seq_num).toBeGreaterThanOrEqual( + if (session.nextReadPosition()) { + expect(session.nextReadPosition()?.seq_num).toBeGreaterThanOrEqual( record.seq_num, ); } @@ -113,8 +113,8 @@ describe("ReadSession Integration Tests", () => { } // After reading, streamPosition should be set - expect(session.lastReadPosition()).toBeDefined(); - expect(session.lastReadPosition()?.seq_num).toBeGreaterThan(0); + expect(session.nextReadPosition()).toBeDefined(); + expect(session.nextReadPosition()?.seq_num).toBeGreaterThan(0); }, ); diff --git a/src/tests/retryAppendSession.test.ts b/src/tests/retryAppendSession.test.ts new file mode 100644 index 0000000..800c3fc --- /dev/null +++ b/src/tests/retryAppendSession.test.ts @@ -0,0 +1,367 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { S2Error } from "../error.js"; +import type { AppendAck, StreamPosition } from "../generated/index.js"; +import type { AppendResult, CloseResult } from "../lib/result.js"; +import { err, errClose, ok, okClose } from "../lib/result.js"; +import { RetryAppendSession } from "../lib/retry.js"; +import type { + AcksStream, + AppendArgs, + AppendRecord, + AppendSession, + TransportAppendSession, +} from "../lib/stream/types.js"; + +/** + * Minimal controllable AppendSession for testing RetryAppendSession. + */ +class FakeAppendSession implements AppendSession { + public readonly readable: ReadableStream; + public readonly writable: WritableStream; + private acksController!: ReadableStreamDefaultController; + private closed = false; + public writes: AppendArgs[] = []; + + failureCause(): undefined { + return undefined; + } + + constructor( + private readonly behavior: { + rejectWritesWith?: S2Error; // if provided, writer.write rejects with this error + neverAck?: boolean; // if true, never emit acks + errorAcksWith?: S2Error; // if provided, acks() stream errors after first write + } = {}, + ) { + this.readable = new ReadableStream({ + start: (c) => { + this.acksController = c; + }, + }); + + this.writable = new WritableStream({ + write: async (args) => { + if (this.closed) { + throw new S2Error({ message: "AppendSession is closed" }); + } + if (this.behavior.rejectWritesWith) { + throw this.behavior.rejectWritesWith; + } + this.writes.push(args); + + // Optionally error the acks stream right after a write + if (this.behavior.errorAcksWith) { + queueMicrotask(() => { + try { + this.acksController.error(this.behavior.errorAcksWith); + } catch {} + }); + } + + // Optionally emit an ack immediately + if (!this.behavior.neverAck && !this.behavior.errorAcksWith) { + const count = Array.isArray(args.records) ? args.records.length : 1; + const start = { seq_num: 0, timestamp: 0 } as StreamPosition; + const end = { seq_num: count, timestamp: 0 } as StreamPosition; + const tail = { seq_num: count, timestamp: 0 } as StreamPosition; + const ack: AppendAck = { start, end, tail }; + this.acksController.enqueue(ack); + } + }, + close: async () => { + this.closed = true; + try { + this.acksController.close(); + } catch {} + }, + abort: async (reason) => { + this.closed = true; + try { + this.acksController.error( + reason instanceof S2Error + ? reason + : new S2Error({ message: String(reason) }), + ); + } catch {} + }, + }); + } + + acks(): AcksStream { + return this.readable as AcksStream; + } + + async close(): Promise { + await this.writable.close(); + } + + async [Symbol.asyncDispose](): Promise { + await this.close(); + } + + submit( + records: AppendRecord | AppendRecord[], + _args?: Omit & { precalculatedSize?: number }, + ): Promise { + const writer = this.writable.getWriter(); + const batch = Array.isArray(records) ? records : [records]; + return writer.write({ records: batch } as AppendArgs) as any; + } + + lastAckedPosition(): AppendAck | undefined { + return undefined; + } +} + +/** + * Transport-level fake session that returns discriminated unions. + * Used for testing RetryAppendSession which wraps transport sessions. + */ +class FakeTransportAppendSession implements TransportAppendSession { + public writes: Array<{ records: AppendRecord[]; args?: any }> = []; + private closed = false; + private ackIndex = 0; + + constructor( + private readonly behavior: { + submitError?: S2Error; // if provided, submit() returns error result + closeError?: S2Error; // if provided, close() returns error result + neverAck?: boolean; // if true, submit() hangs forever (for timeout tests) + customAcks?: AppendAck[]; // if provided, return these acks in sequence + } = {}, + ) {} + + async submit( + records: AppendRecord | AppendRecord[], + args?: Omit & { precalculatedSize?: number }, + ): Promise { + if (this.closed) { + return err(new S2Error({ message: "session is closed", status: 400 })); + } + + if (this.behavior.submitError) { + return err(this.behavior.submitError); + } + + if (this.behavior.neverAck) { + // Hang forever (for timeout tests) + return new Promise(() => {}); + } + + const batch = Array.isArray(records) ? records : [records]; + this.writes.push({ records: batch, args }); + + // Return custom ack if provided + if ( + this.behavior.customAcks && + this.ackIndex < this.behavior.customAcks.length + ) { + const ack = this.behavior.customAcks[this.ackIndex++]!; + return ok(ack); + } + + // Return default successful ack + const count = batch.length; + const start = { seq_num: 0, timestamp: 0 } as StreamPosition; + const end = { seq_num: count, timestamp: 0 } as StreamPosition; + const tail = { seq_num: count, timestamp: 0 } as StreamPosition; + const ack: AppendAck = { start, end, tail }; + return ok(ack); + } + + async close(): Promise { + if (this.behavior.closeError) { + return errClose(this.behavior.closeError); + } + this.closed = true; + return okClose(); + } +} + +describe("RetryAppendSession (unit)", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("aborts on ack timeout (~5s from enqueue) when no acks arrive", async () => { + const session = await RetryAppendSession.create(async () => { + // Accept writes but never emit acks + return new FakeTransportAppendSession({ neverAck: true }); + }); + (session as any).requestTimeoutMs = 500; + + const ackP = session.submit([{ body: "x" }]); + + // Not yet timed out at 0.49s + await vi.advanceTimersByTimeAsync(490); + await Promise.resolve(); + let settled = false; + ackP.then(() => (settled = true)).catch(() => (settled = true)); + await Promise.resolve(); + expect(settled).toBe(false); + + // Time out after ~0.5s + await vi.advanceTimersByTimeAsync(20); + await Promise.resolve(); + await expect(ackP).rejects.toMatchObject({ status: 408 }); + }); + + it("recovers from send-phase transient error and resolves after recovery", async () => { + // First session rejects writes; second accepts and acks immediately + let call = 0; + const session = await RetryAppendSession.create( + async () => { + call++; + if (call === 1) { + return new FakeTransportAppendSession({ + submitError: new S2Error({ message: "boom", status: 500 }), + }); + } + return new FakeTransportAppendSession(); + }, + undefined, + { retryBackoffDurationMs: 1, maxAttempts: 2, appendRetryPolicy: "all" }, + ); + + const p = session.submit([{ body: "x" }]); + // Allow microtasks (acks error propagation) to run + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(10); + await Promise.resolve(); + const ack = await p; + expect(ack.end.seq_num - ack.start.seq_num).toBe(1); + }); + + it("fails immediately when retries are disabled and send-phase errors persist", async () => { + const error = new S2Error({ message: "boom", status: 500 }); + const session = await RetryAppendSession.create( + async () => new FakeTransportAppendSession({ submitError: error }), + undefined, + { retryBackoffDurationMs: 1, maxAttempts: 0, appendRetryPolicy: "all" }, + ); + + const ackP = session.submit([{ body: "x" }]); + await expect(ackP).rejects.toMatchObject({ + message: "Max retry attempts (0) exceeded: boom", + status: 500, + }); + }); + + it("does not retry non-idempotent inflight under noSideEffects policy and exposes failure cause", async () => { + const error = new S2Error({ message: "boom", status: 500 }); + const session = await RetryAppendSession.create( + async () => new FakeTransportAppendSession({ submitError: error }), + undefined, + { + retryBackoffDurationMs: 1, + maxAttempts: 2, + appendRetryPolicy: "noSideEffects", + }, + ); + + const p1 = session.submit([{ body: "x" }]); + await expect(p1).rejects.toMatchObject({ status: 500 }); + expect(session.failureCause()).toMatchObject({ status: 500 }); + }); + + it("abort rejects backlog and queued submissions with the abort error", async () => { + const error = new S2Error({ message: "boom", status: 500 }); + const session = await RetryAppendSession.create( + async () => new FakeTransportAppendSession({ submitError: error }), + undefined, + { + retryBackoffDurationMs: 1, + maxAttempts: 2, + appendRetryPolicy: "noSideEffects", + }, + ); + + const p1 = session.submit([{ body: "a" }]); + const p2 = session.submit([{ body: "b" }]); + await expect(p1).rejects.toMatchObject({ status: 500 }); + await expect(p2).rejects.toMatchObject({ status: 500 }); + }); + + it("detects non-monotonic sequence numbers and aborts with fatal error", async () => { + // Create acks with non-monotonic sequence numbers + // Each ack must have correct count (end - start = 1 for single record batches) + const ack1: AppendAck = { + start: { seq_num: 0, timestamp: 0 }, + end: { seq_num: 1, timestamp: 0 }, // count = 1 + tail: { seq_num: 1, timestamp: 0 }, + }; + const ack2: AppendAck = { + start: { seq_num: 0, timestamp: 0 }, // Decreasing! + end: { seq_num: 1, timestamp: 0 }, + tail: { seq_num: 1, timestamp: 0 }, + }; + + const session = await RetryAppendSession.create( + async () => new FakeTransportAppendSession({ customAcks: [ack1, ack2] }), + undefined, + { retryBackoffDurationMs: 1, maxAttempts: 0 }, // No retries + ); + + // First submit should succeed + const p1 = session.submit([{ body: "a" }]); + await expect(p1).resolves.toMatchObject({ end: { seq_num: 1 } }); + + // Second submit should trigger invariant violation + const p2 = session.submit([{ body: "b" }]); + await expect(p2).rejects.toMatchObject({ + message: expect.stringContaining( + "Sequence number not strictly increasing", + ), + status: 500, + code: "INTERNAL_ERROR", + }); + + // Session should expose the failure cause + expect(session.failureCause()).toMatchObject({ + message: expect.stringContaining( + "Sequence number not strictly increasing", + ), + status: 500, + }); + + // Subsequent submits should also fail + const p3 = session.submit([{ body: "c" }]); + await expect(p3).rejects.toMatchObject({ status: 500 }); + }); + + it("detects non-increasing (equal) sequence numbers and aborts", async () => { + // Create acks with equal sequence numbers + // Each ack must have correct count (end - start = 1 for single record batches) + const ack1: AppendAck = { + start: { seq_num: 9, timestamp: 0 }, + end: { seq_num: 10, timestamp: 0 }, // count = 1 + tail: { seq_num: 10, timestamp: 0 }, + }; + const ack2: AppendAck = { + start: { seq_num: 9, timestamp: 0 }, + end: { seq_num: 10, timestamp: 0 }, // Equal end, not increasing! + tail: { seq_num: 10, timestamp: 0 }, + }; + + const session = await RetryAppendSession.create( + async () => new FakeTransportAppendSession({ customAcks: [ack1, ack2] }), + undefined, + { retryBackoffDurationMs: 1, maxAttempts: 0 }, + ); + + // First submit should succeed + await expect(session.submit([{ body: "a" }])).resolves.toMatchObject({ + end: { seq_num: 10 }, + }); + + // Second submit should trigger invariant violation + const error = await session.submit([{ body: "b" }]).catch((e) => e); + expect(error.message).toContain("Sequence number not strictly increasing"); + expect(error.message).toContain("previous=10"); + expect(error.message).toContain("current=10"); + expect(error.status).toBe(500); + }); +}); diff --git a/src/tests/retryReadSession.test.ts b/src/tests/retryReadSession.test.ts new file mode 100644 index 0000000..cca5133 --- /dev/null +++ b/src/tests/retryReadSession.test.ts @@ -0,0 +1,445 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { S2Error } from "../error.js"; +import type { StreamPosition } from "../generated/index.js"; +import { RetryReadSession } from "../lib/retry.js"; +import type { + ReadArgs, + ReadRecord, + ReadResult, + TransportReadSession, +} from "../lib/stream/types.js"; + +/** + * Fake TransportReadSession for testing RetryReadSession. + * Implements the transport layer pattern: yields ReadResult and never throws. + */ +class FakeReadSession + extends ReadableStream> + implements TransportReadSession +{ + public recordsEmitted = 0; + + constructor( + private readonly behavior: { + // Records to emit before erroring (if errorAfterRecords is set) + records: Array>; + // Error after emitting this many records (undefined = no error) + errorAfterRecords?: number; + // Error to emit as error result + error?: S2Error; + }, + ) { + let emittedCount = 0; // Use local variable in super() callback + super({ + pull: (controller) => { + // Check if we should error before emitting any more records + if ( + behavior.errorAfterRecords !== undefined && + emittedCount >= behavior.errorAfterRecords + ) { + // Emit error result instead of throwing + controller.enqueue({ + ok: false, + error: + behavior.error ?? new S2Error({ message: "boom", status: 500 }), + }); + controller.close(); + return; + } + + // Emit records one at a time as they're requested + if (emittedCount < behavior.records.length) { + // Emit success result + controller.enqueue({ + ok: true, + value: behavior.records[emittedCount]!, + }); + emittedCount++; + + // Check if we should error after emitting this record + if ( + behavior.errorAfterRecords !== undefined && + emittedCount >= behavior.errorAfterRecords + ) { + // Emit error result instead of throwing + controller.enqueue({ + ok: false, + error: + behavior.error ?? new S2Error({ message: "boom", status: 500 }), + }); + controller.close(); + return; + } + } else { + // All records emitted + controller.close(); + } + }, + }); + this.recordsEmitted = behavior.errorAfterRecords ?? behavior.records.length; + } + + nextReadPosition(): StreamPosition | undefined { + if (this.recordsEmitted === 0) return undefined; + const lastRecord = this.behavior.records[this.recordsEmitted - 1]; + if (!lastRecord) return undefined; + return { + seq_num: lastRecord.seq_num + 1, + timestamp: lastRecord.timestamp, + }; + } + + lastObservedTail(): StreamPosition | undefined { + return undefined; + } + + // Implement AsyncIterable (for await...of support) + [Symbol.asyncIterator](): AsyncIterableIterator> { + const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; + if (typeof fn === "function") return fn.call(this); + const reader = this.getReader(); + return { + next: async () => { + const r = await reader.read(); + if (r.done) return { done: true, value: undefined }; + return { done: false, value: r.value }; + }, + return: async (value?: any) => { + reader.releaseLock(); + return { done: true, value }; + }, + throw: async (e?: any) => { + reader.releaseLock(); + throw e; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + } + + // Implement AsyncDisposable (using Disposable) + async [Symbol.asyncDispose](): Promise { + await this.cancel(); + } +} + +describe("RetryReadSession (unit)", () => { + // Note: Not using fake timers here because they don't play well with async iteration + // Instead, we use very short backoff times (1ms) to make tests run fast + + it("adjusts count parameter on retry after partial read", async () => { + const records: ReadRecord<"string">[] = [ + { seq_num: 0, timestamp: 0, body: "a" }, + { seq_num: 1, timestamp: 0, body: "b" }, + { seq_num: 2, timestamp: 0, body: "c" }, + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 3 records then error + return new FakeReadSession({ + records, + errorAfterRecords: 3, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: emit remaining records (none in this case, but succeed) + return new FakeReadSession({ records: [] }); + }, + { count: 10 }, // Request 10 records + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got all 3 records (transport explicitly emits them as success before error) + expect(results).toHaveLength(3); + + // Verify retry adjusted count: 10 - 3 = 7 + expect(capturedArgs).toHaveLength(2); + expect(capturedArgs[0]?.count).toBe(10); + expect(capturedArgs[1]?.count).toBe(7); + }); + + it("adjusts bytes parameter on retry after partial read", async () => { + // Each record is ~50 bytes (rough estimate with body + overhead) + const records: ReadRecord<"string">[] = [ + { seq_num: 0, timestamp: 0, body: "x".repeat(42) }, // ~50 bytes + { seq_num: 1, timestamp: 0, body: "y".repeat(42) }, // ~50 bytes + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 2 records (~100 bytes) then error + return new FakeReadSession({ + records, + errorAfterRecords: 2, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: succeed + return new FakeReadSession({ records: [] }); + }, + { bytes: 500 }, // Request 500 bytes + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got both records (transport explicitly emits them as success before error) + expect(results).toHaveLength(2); + + // Verify retry adjusted bytes: should be less than 500 + expect(capturedArgs).toHaveLength(2); + expect(capturedArgs[0]?.bytes).toBe(500); + expect(capturedArgs[1]?.bytes).toBeLessThan(500); + // Each record is approximately 50 bytes, so should be around 500 - 100 = 400 + expect(capturedArgs[1]?.bytes).toBeGreaterThan(350); + }); + + it("adjusts wait parameter based on elapsed time", async () => { + const records: ReadRecord<"string">[] = [ + { seq_num: 0, timestamp: 0, body: "a" }, + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 1 record then error + return new FakeReadSession({ + records, + errorAfterRecords: 1, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: succeed + return new FakeReadSession({ records: [] }); + }, + { wait: 10 }, // Wait up to 10 seconds + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got the record (transport explicitly emits it as success before error) + expect(results).toHaveLength(1); + + // Verify retry adjusted wait: should be less than original 10 seconds + expect(capturedArgs).toHaveLength(2); + expect(capturedArgs[0]?.wait).toBe(10); + // Should be less than original 10 seconds (since some time elapsed) + expect(capturedArgs[1]?.wait).toBeLessThan(10); + expect(capturedArgs[1]?.wait).toBeGreaterThanOrEqual(0); + }); + + it("adjusts seq_num to resume from next position on retry", async () => { + const records: ReadRecord<"string">[] = [ + { seq_num: 100, timestamp: 0, body: "a" }, + { seq_num: 101, timestamp: 0, body: "b" }, + { seq_num: 102, timestamp: 0, body: "c" }, + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 3 records (seq_num 100-102) then error + return new FakeReadSession({ + records, + errorAfterRecords: 3, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: succeed + return new FakeReadSession({ records: [] }); + }, + { seq_num: 100 }, // Start from seq_num 100 + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records (including through retry) + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got all 3 records (transport explicitly emits them as success before error) + expect(results).toHaveLength(3); + + // Verify retry adjusted seq_num to 103 (102 + 1) + expect(capturedArgs).toHaveLength(2); + expect(capturedArgs[0]?.seq_num).toBe(100); + expect(capturedArgs[1]?.seq_num).toBe(103); + }); + + it("does not adjust until parameter on retry (absolute boundary)", async () => { + const records: ReadRecord<"string">[] = [ + { seq_num: 0, timestamp: 0, body: "a" }, + { seq_num: 1, timestamp: 0, body: "b" }, + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 2 records then error + return new FakeReadSession({ + records, + errorAfterRecords: 2, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: succeed + return new FakeReadSession({ records: [] }); + }, + { until: 1000 }, // Read until seq_num 1000 + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got both records (transport explicitly emits them as success before error) + expect(results).toHaveLength(2); + + // Verify until remains unchanged (it's an absolute boundary) + expect(capturedArgs).toHaveLength(2); + expect(capturedArgs[0]?.until).toBe(1000); + expect(capturedArgs[1]?.until).toBe(1000); + }); + + it("combines all parameter adjustments on retry", async () => { + const records: ReadRecord<"string">[] = [ + { seq_num: 50, timestamp: 0, body: "x".repeat(42) }, + { seq_num: 51, timestamp: 0, body: "y".repeat(42) }, + ]; + + let callCount = 0; + const capturedArgs: Array> = []; + + const session = await RetryReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + callCount++; + if (callCount === 1) { + // First call: emit 2 records then error + return new FakeReadSession({ + records, + errorAfterRecords: 2, + error: new S2Error({ message: "transient error", status: 500 }), + }); + } + // Second call: succeed + return new FakeReadSession({ records: [] }); + }, + { + seq_num: 50, + count: 10, + bytes: 500, + wait: 30, + until: 1000, + }, + { retryBackoffDurationMs: 1, maxAttempts: 1 }, + ); + + // Consume all records + const results: ReadRecord<"string">[] = []; + for await (const record of session) { + results.push(record); + } + + // Verify we got both records (transport explicitly emits them as success before error) + expect(results).toHaveLength(2); + + // Verify all adjustments + expect(capturedArgs).toHaveLength(2); + const firstArgs = capturedArgs[0]!; + const secondArgs = capturedArgs[1]!; + + // Original args + expect(firstArgs.seq_num).toBe(50); + expect(firstArgs.count).toBe(10); + expect(firstArgs.bytes).toBe(500); + expect(firstArgs.wait).toBe(30); + expect(firstArgs.until).toBe(1000); + + // Adjusted args + expect(secondArgs.seq_num).toBe(52); // 50 + 2 (read 2 records) + expect(secondArgs.count).toBe(8); // 10 - 2 + expect(secondArgs.bytes).toBeLessThan(500); // Decremented by ~100 + expect(secondArgs.bytes).toBeGreaterThan(350); // Should be around 400 + expect(secondArgs.wait).toBeLessThan(30); // Decremented by elapsed time + expect(secondArgs.until).toBe(1000); // Unchanged (absolute boundary) + }); + + it("fails after max retry attempts exhausted", async () => { + let callCount = 0; + + const session = await RetryReadSession.create( + async (_args) => { + callCount++; + // Always error immediately without emitting any successful records + return new FakeReadSession({ + records: [], + errorAfterRecords: 0, + error: new S2Error({ message: "persistent error", status: 500 }), + }); + }, + { count: 10 }, + { retryBackoffDurationMs: 1, maxAttempts: 2 }, // Allow 2 retries + ); + + // Try to consume the stream - should fail after exhausting retries + await expect(async () => { + for await (const _record of session) { + // Should eventually fail + } + }).rejects.toMatchObject({ + message: expect.stringContaining("persistent error"), + }); + + // Should have tried 3 times (initial + 2 retries) + expect(callCount).toBe(3); + }); +}); diff --git a/src/utils.ts b/src/utils.ts index 71330e3..642da17 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,7 @@ import type { AppendHeaders, AppendRecord as AppendRecordType, + ReadRecord, } from "./lib/stream/types.js"; export type AppendRecord = AppendRecordType; @@ -136,7 +137,9 @@ export function utf8ByteLength(str: string): number { * @param record The record to measure * @returns The size in bytes */ -export function meteredSizeBytes(record: AppendRecord): number { +export function meteredSizeBytes( + record: AppendRecord | ReadRecord, +): number { // Calculate header size based on actual data types let numHeaders = 0; let headersSize = 0; From ff8f947022dd3ae76a2dae07a3cc7cedef5ca686 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 21:33:24 -0800 Subject: [PATCH 05/26] a --- src/lib/retry.ts.old | 1408 ------------------------- src/lib/stream/transport/s2s/index.ts | 13 +- 2 files changed, 2 insertions(+), 1419 deletions(-) delete mode 100644 src/lib/retry.ts.old diff --git a/src/lib/retry.ts.old b/src/lib/retry.ts.old deleted file mode 100644 index 94ffec5..0000000 --- a/src/lib/retry.ts.old +++ /dev/null @@ -1,1408 +0,0 @@ -import createDebug from "debug"; -import type { RetryConfig } from "../common.js"; -import { S2Error, s2Error, withS2Error } from "../error.js"; -import type { AppendAck, StreamPosition } from "../generated/index.js"; -import { meteredSizeBytes } from "../utils.js"; -import type { - AcksStream, - AppendArgs, - AppendRecord, - AppendSession, - AppendSessionOptions, - ReadArgs, - ReadRecord, - ReadSession, -} from "./stream/types.js"; -import type { AppendResult, CloseResult } from "./result.js"; -import { ok, err, okClose, errClose } from "./result.js"; - -const debug = createDebug("s2:retry"); - -/** - * Default retry configuration. - */ -export const DEFAULT_RETRY_CONFIG: Required & { - requestTimeoutMs: number; -} = { - maxAttempts: 3, - retryBackoffDurationMs: 100, - appendRetryPolicy: "noSideEffects", - requestTimeoutMs: 5000, // 5 seconds -}; - -const RETRYABLE_STATUS_CODES = new Set([ - 408, // request_timeout - 429, // too_many_requests - 500, // internal_server_error - 502, // bad_gateway - 503, // service_unavailable -]); - -/** - * Determines if an error should be retried based on its characteristics. - * 400-level errors (except 408, 429) are non-retryable validation/client errors. - */ -export function isRetryable(error: S2Error): boolean { - if (!error.status) return false; - - // Explicit retryable codes (including some 4xx like 408, 429) - if (RETRYABLE_STATUS_CODES.has(error.status)) { - return true; - } - - // 400-level errors are generally non-retryable (validation, bad request) - if (error.status >= 400 && error.status < 500) { - return false; - } - - return false; -} - -/** - * Calculates the delay before the next retry attempt using exponential backoff. - */ -export function calculateDelay(attempt: number, baseDelayMs: number): number { - // Exponential backoff: baseDelay * (2 ^ attempt) - const delay = baseDelayMs * Math.pow(2, attempt); - // Add jitter: random value between 0 and delay - const jitter = Math.random() * delay; - return Math.floor(delay + jitter); -} - -/** - * Sleeps for the specified duration. - */ -export function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** - * Executes an async function with automatic retry logic for transient failures. - * - * @param retryConfig Retry configuration (max attempts, backoff duration) - * @param fn The async function to execute - * @returns The result of the function - * @throws The last error if all retry attempts are exhausted - */ -export async function withRetries( - retryConfig: RetryConfig | undefined, - fn: () => Promise, - isPolicyCompliant: (config: RetryConfig, error: S2Error) => boolean = () => - true, -): Promise { - const config = { - ...DEFAULT_RETRY_CONFIG, - ...retryConfig, - }; - - // If maxAttempts is 0, don't retry at all - if (config.maxAttempts === 0) { - debug("maxAttempts is 0, retries disabled"); - return fn(); - } - - let lastError: S2Error | undefined = undefined; - - for (let attempt = 0; attempt <= config.maxAttempts; attempt++) { - try { - const result = await fn(); - if (attempt > 0) { - debug("succeeded after %d retries", attempt); - } - return result; - } catch (error) { - // withRetry only handles S2Errors (withS2Error should be called first) - if (!(error instanceof S2Error)) { - debug("non-S2Error thrown, rethrowing immediately: %s", error); - throw error; - } - - lastError = error; - - // Don't retry if this is the last attempt - if (attempt === config.maxAttempts) { - debug("max attempts exhausted, throwing error"); - break; - } - - // Check if error is retryable - if (!isPolicyCompliant(config, lastError) || !isRetryable(lastError)) { - debug("error not retryable, throwing immediately"); - throw error; - } - - // Calculate delay and wait before retrying - const delay = calculateDelay(attempt, config.retryBackoffDurationMs); - debug( - "retryable error, backing off for %dms, status=%s", - delay, - error.status, - ); - await sleep(delay); - } - } - - throw lastError; -} -export class RetryReadSession - extends ReadableStream> - implements ReadSession -{ - private _nextReadPosition: StreamPosition | undefined = undefined; - - private _recordsRead: number = 0; - private _bytesRead: number = 0; - - static async create( - generator: (args: ReadArgs) => Promise>, - args: ReadArgs = {}, - config?: RetryConfig, - ) { - return new RetryReadSession(args, generator, config); - } - - private constructor( - args: ReadArgs, - generator: (args: ReadArgs) => Promise>, - config?: RetryConfig, - ) { - const retryConfig = { - ...DEFAULT_RETRY_CONFIG, - ...config, - }; - let session: ReadSession | undefined = undefined; - super({ - start: async (controller) => { - let nextArgs = { ...args }; - let attempt = 0; - - while (true) { - try { - session = await generator(nextArgs); - const reader = session.getReader(); - - while (true) { - const { done, value } = await reader.read(); - attempt = 0; - if (done) { - break; - } - this._nextReadPosition = { - seq_num: value.seq_num + 1, - timestamp: value.timestamp, - }; - this._recordsRead++; - this._bytesRead += meteredSizeBytes(value); - - controller.enqueue(value); - } - reader.releaseLock(); - break; - } catch (e) { - let error = s2Error(e); - if (isRetryable(error) && attempt < retryConfig.maxAttempts) { - if (this._nextReadPosition) { - nextArgs.seq_num = this._nextReadPosition.seq_num; - } - if (nextArgs.count) { - nextArgs.count = - this._recordsRead === undefined - ? nextArgs.count - : nextArgs.count - this._recordsRead; - } - if (nextArgs.bytes) { - nextArgs.bytes = - this._bytesRead === undefined - ? nextArgs.bytes - : nextArgs.bytes - this._bytesRead; - } - // TODO also correct wait - const delay = calculateDelay( - attempt, - retryConfig.retryBackoffDurationMs, - ); - debug("will retry after %dms, status=%s", delay, error.status); - await sleep(delay); - attempt++; - continue; - } - - debug("error in retry loop: %s", e); - throw error; - } - } - - controller.close(); - }, - cancel: async () => { - session?.cancel(); - }, - }); - } - - async [Symbol.asyncDispose]() { - await this.cancel("disposed"); - } - - // Polyfill for older browsers / Node.js environments - [Symbol.asyncIterator](): AsyncIterableIterator> { - const fn = (ReadableStream.prototype as any)[Symbol.asyncIterator]; - if (typeof fn === "function") return fn.call(this); - const reader = this.getReader(); - return { - next: async () => { - const r = await reader.read(); - if (r.done) { - reader.releaseLock(); - return { done: true, value: undefined }; - } - return { done: false, value: r.value }; - }, - throw: async (e) => { - await reader.cancel(e); - reader.releaseLock(); - return { done: true, value: undefined }; - }, - return: async () => { - await reader.cancel("done"); - reader.releaseLock(); - return { done: true, value: undefined }; - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - } - - lastObservedTail(): StreamPosition | undefined { - return undefined; - } - - nextReadPosition(): StreamPosition | undefined { - return undefined; - } -} - -/** - * RetryAppendSession wraps an underlying AppendSession with automatic retry logic. - * - * Architecture: - * - All writes (submit() and writable.write()) are serialized through inflightQueue - * - inflightQueue tracks batches that have been submitted but not yet acked - * - Background ack reader consumes acks and matches them FIFO with inflightQueue - * - On error, _initSession() recreates session and re-transmits all inflightQueue batches - * - Ack timeout is fatal: if no ack arrives within the timeout window, - * the session aborts and rejects queued writers - * - * Flow for a successful append: - * 1. submit(records) adds batch to inflightQueue with promise resolvers - * 2. Calls underlying session.submit() to send batch - * 3. Background reader receives ack, validates record count - * 4. Resolves promise, removes from inflightQueue, forwards ack to user - * - * Flow for a failed append: - * 1. submit(records) adds batch to inflightQueue - * 2. Calls underlying session.submit() which fails - * 3. Checks if retryable (status code, retry policy, idempotency) - * 4. Calls _initSession() which closes old session, creates new session - * 5. _initSession() re-transmits ALL batches in inflightQueue (recovery) - * 6. Background reader receives acks for recovered batches - * 7. Original submit() call's promise is resolved by background reader - * - * Invariants: - * - Exactly one ack per batch in FIFO order - * - Ack record count matches batch record count - * - Acks arrive within ackTimeoutMs (5s) or session is retried - */ -class AsyncQueue { - private values: T[] = []; - private waiters: Array<(value: T) => void> = []; - - push(value: T): void { - const waiter = this.waiters.shift(); - if (waiter) { - waiter(value); - return; - } - this.values.push(value); - } - - async next(): Promise { - if (this.values.length > 0) { - return this.values.shift()!; - } - return new Promise((resolve) => { - this.waiters.push(resolve); - }); - } - - clear(): void { - this.values = []; - this.waiters = []; - } - - // Drain currently buffered values (non-blocking) and clear the buffer. - drain(): T[] { - const out = this.values; - this.values = []; - return out; - } -} - -/** - * New simplified inflight entry for the pump-based architecture. - * Each entry tracks a batch and its promise from the inner transport session. - */ -type InflightEntry = { - records: AppendRecord[]; - args?: Omit & { precalculatedSize?: number }; - expectedCount: number; - meteredBytes: number; - enqueuedAt: number; // Timestamp for timeout anchoring - innerPromise: Promise; // Promise from transport session - maybeResolve?: (result: AppendResult) => void; // Resolver for submit() callers -}; - -const DEFAULT_MAX_QUEUED_BYTES = 10 * 1024 * 1024; // 10 MiB default - -export class RetryAppendSession implements AppendSession, AsyncDisposable { - private readonly ackTimeoutMs = 10000; - private readonly maxQueuedBytes: number; - private readonly retryConfig: Required; - - private readonly notificationQueue = new AsyncQueue(); - private readonly inflight: InflightEntry[] = []; - private readonly capacityWaiters: Array<() => void> = []; - - private session: AppendSession | undefined = undefined; - private sessionWriter?: WritableStreamDefaultWriter; - private sessionReader?: ReadableStreamDefaultReader; - - private queuedBytes = 0; - private pendingBytes = 0; - private consecutiveFailures = 0; - - private pumpPromise?: Promise; - private pumpStopped = false; - private closePromise?: Promise; - private closeResolve?: () => void; - private closeReject?: (error: S2Error) => void; - private closing = false; - private pumpError?: S2Error; - private readerToken?: object; - private readerTokenId?: number; - private readerTokenGen: number = 0; - private readonly readerTokenIds = new WeakMap(); - private ackTimeoutHandle?: ReturnType; - private ackTimeoutToken?: object; - private ackTimeoutHead?: InflightEntry; - // removed recoveryPromise in favor of explicit flags - private closed = false; - - // Recovery gating - private pausedForRecovery: boolean = false; - private recoveryBarrierLeft: number = 0; - private backlog: InflightEntry[] = []; - private recovering: boolean = false; - private restartRequested: boolean = false; - private maxInflightBatches?: number; - private reservedBatches: number = 0; - - private _lastAckedPosition?: AppendAck; - private acksController?: ReadableStreamDefaultController; - - public readonly readable: ReadableStream; - public readonly writable: WritableStream; - - /** - * If the session has failed, returns the original fatal error that caused - * the pump to stop. Returns undefined when the session has not failed. - */ - failureCause(): S2Error | undefined { - return this.pumpError; - } - - constructor( - private readonly generator: ( - options?: AppendSessionOptions, - ) => Promise, - private readonly sessionOptions?: AppendSessionOptions, - config?: RetryConfig, - ) { - this.retryConfig = { - ...DEFAULT_RETRY_CONFIG, - ...config, - }; - this.maxQueuedBytes = - this.sessionOptions?.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES; - this.maxInflightBatches = this.sessionOptions?.maxInflightBatches; - - this.readable = new ReadableStream({ - start: (controller) => { - this.acksController = controller; - }, - }); - - this.writable = new WritableStream({ - write: async (chunk) => { - const recordsArray = Array.isArray(chunk.records) - ? chunk.records - : [chunk.records]; - const args = { ...chunk } as Omit & { - precalculatedSize?: number; - }; - delete (args as any).records; - const { ackPromise, enqueuedPromise } = await this.enqueueBatch( - recordsArray, - args, - ); - ackPromise.catch(() => { - // Swallow to avoid unhandled rejection; pump already surfaces the error. - }); - return enqueuedPromise; - }, - close: async () => { - await this.close(); - }, - abort: async (reason) => { - await this.abort(reason); - }, - }); - } - - static async create( - generator: ( - options?: AppendSessionOptions, - ) => Promise, - sessionOptions?: AppendSessionOptions, - config?: RetryConfig, - ): Promise { - return new RetryAppendSession(generator, sessionOptions, config); - } - - private ensurePump(): void { - if (this.pumpPromise) { - return; - } - this.pumpPromise = this.runPump(); - } - - private async runPump(): Promise { - try { - while (true) { - if (this.pumpStopped) { - break; - } - if (this.shouldAttemptClose()) { - await this.finishClose(); - continue; - } - const notification = await this.notificationQueue.next(); - if (notification.type === "stop") { - break; - } - await this.processNotification(notification); - } - } catch (error) { - this.setPumpError(s2Error(error)); - } finally { - this.closed = true; - this.pumpStopped = true; - this.resolveClosePromise(); - } - } - - private shouldAttemptClose(): boolean { - const ok = - this.closing && - !this.pumpStopped && - this.inflight.length === 0 && - !this.pausedForRecovery && - !this.recovering && - this.backlog.length === 0; - if (this.closing) { - debug( - "shouldAttemptClose? %s (inflight=%d paused=%s recovering=%s backlog=%d)", - ok, - this.inflight.length, - this.pausedForRecovery, - this.recovering, - this.backlog.length, - ); - } - return ok; - } - - private async processNotification(notification: Notification): Promise { - const tokId = (notification as any).token - ? this.readerTokenIds.get((notification as any).token) - : undefined; - debug( - "process: inflight=%d type=%s tokenId=%s", - this.inflight.length, - notification.type, - tokId ?? "none", - ); - switch (notification.type) { - case "batch": - await this.handleBatch(notification.entry); - break; - case "ack": - if (notification.token && notification.token !== this.readerToken) { - break; - } - await this.handleAck(notification.ack); - break; - case "error": - if (notification.token && notification.token !== this.readerToken) { - break; - } - await this.startRecovery(notification.error); - break; - case "close": - debug("close"); - await this.handleClose(notification); - break; - case "abort": - debug("abort"); - await this.handleAbort(notification.error); - break; - case "stop": - return; - } - } - - private async handleBatch(entry: InflightEntry): Promise { - this.pendingBytes = Math.max(0, this.pendingBytes - entry.meteredBytes); - // Consume a reserved batch slot if batch-based gating is enabled - if (this.maxInflightBatches && this.maxInflightBatches > 0 && this.reservedBatches > 0) { - this.reservedBatches -= 1; - } - - // If recovering (gated), backlog the entry and do not send yet. - if (this.pausedForRecovery) { - this.backlog.push(entry); - this.queuedBytes += entry.meteredBytes; - // Do not arm head timer here; we only arm when actually sending. - this.releaseCapacity(); - return; - } - - // Normal path: inflight + send - this.inflight.push(entry); - this.queuedBytes += entry.meteredBytes; - this.releaseCapacity(); - - try { - await this.ensureSession(); - await this.sendEntry(entry); - } catch (error) { - const err = s2Error(error); - entry.sent = false; - debug("Error sending batch: %s", err.message); - // Respect append retry policy: only retry when allowed - if (!isRetryable(err) || !this.isAppendRetryAllowed(entry)) { - this.removeInflightEntry(entry); - entry.reject(err); - this.setPumpError(err); - return; - } - this.notificationQueue.push({ type: "error", error: err }); - } - } - - private async handleAck(ack: AppendAck): Promise { - if (this.inflight.length === 0) { - const fatal = new S2Error({ - message: "Invariant violation: received ack with empty inflight queue", - status: 500, - }); - this.setPumpError(fatal); - return; - } - - const entry = this.inflight.shift()!; - const actualCount = ack.end.seq_num - ack.start.seq_num; - if (actualCount !== entry.expectedCount) { - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - this.releaseCapacity(); - const error = new S2Error({ - message: `Invariant violation: Ack record count mismatch. Expected ${entry.expectedCount}, got ${actualCount}`, - status: 500, - }); - entry.reject(error); - this.setPumpError(error); - return; - } - - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - this.releaseCapacity(); - - this._lastAckedPosition = ack; - this.consecutiveFailures = 0; - - entry.resolve(ack); - if (this.acksController) { - try { - this.acksController.enqueue(ack); - } catch (error) { - debug("Failed to enqueue ack: %s", error); - } - } - - this.startAckTimerForHead(); - - // If recovering, count down acks from the snapshot; when drained, flush backlog and resume. - if (this.pausedForRecovery && this.recoveryBarrierLeft > 0) { - debug( - "recovery ack: remaining=%d inflight_after=%d", - this.recoveryBarrierLeft - 1, - this.inflight.length, - ); - this.recoveryBarrierLeft -= 1; - if (this.recoveryBarrierLeft === 0) { - debug("recovery barrier drained; flushing backlog size=%d", this.backlog.length); - await this.flushBacklog(); - this.pausedForRecovery = false; - this.recovering = false; - debug("recovery complete; resuming writers (waiters=%d)", this.capacityWaiters.length); - // Wake blocked writers - while (this.capacityWaiters.length > 0) { - const resolve = this.capacityWaiters.shift(); - resolve?.(); - } - await this.maybeFinishClose(); - } - } - - await this.maybeFinishClose(); - } - - private async handleClose(_: CloseNotification): Promise { - this.closing = true; - await this.maybeFinishClose(); - } - - private async handleAbort(error: S2Error): Promise { - if (this.pumpStopped) { - return; - } - this.closing = true; - // Cache the original fatal cause if not already recorded - if (!this.pumpError) { - this.pumpError = error; - } - this.failAll(error); - // Reject any backlogged entries that were never sent - if (this.backlog.length > 0) { - for (const entry of this.backlog) { - try { - entry.reject(error); - } catch {} - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - } - this.backlog = []; - } - // Drain and reject any queued-but-not-processed batches. - const drained = this.notificationQueue.drain(); - for (const n of drained) { - if ((n as any).type === "batch") { - const bn = n as any; - this.pendingBytes = Math.max(0, this.pendingBytes - bn.entry.meteredBytes); - try { - bn.entry.reject(error); - } catch {} - } - } - await this.teardownSession(); - // Ensure timeout continues across recovery even if we can't resend immediately. - this.startAckTimerForHead(); - if (this.acksController) { - try { - this.acksController.error(error); - } catch (err) { - debug("Error signaling acks controller during abort: %s", err); - } - } - // Wake all capacity waiters to ensure no writers remain blocked. - while (this.capacityWaiters.length > 0) { - const resolve = this.capacityWaiters.shift(); - try { - resolve?.(); - } catch {} - } - this.stopPump(error); - } - - private async ensureSession(): Promise { - if (this.session && this.sessionWriter && this.sessionReader) { - return; - } - this.session = await this.generator(this.sessionOptions); - this.sessionWriter = this.session.writable.getWriter(); - this.sessionReader = this.session.acks().getReader(); - this.startAckReader(this.sessionReader); - } - - private startAckReader(reader: ReadableStreamDefaultReader): void { - const token = {}; - const id = ++this.readerTokenGen; - this.readerToken = token; - this.readerTokenId = id; - this.readerTokenIds.set(token, id); - debug("reader[%d]: start", id); - - (async () => { - try { - while (true) { - const { done, value } = await reader.read(); - if (this.readerToken !== token) { - break; - } - if (done) { - debug("reader[%d]: done -> enqueue error closed", id); - this.notificationQueue.push({ - type: "error", - error: new S2Error({ - message: "AppendSession reader closed unexpectedly", - status: 500, - }), - token, - }); - break; - } - debug("reader[%d]: ack", id); - this.notificationQueue.push({ type: "ack", ack: value!, token }); - } - } catch (error) { - if (this.readerToken === token) { - debug("reader[%d]: error %s", id, (error as any)?.message ?? String(error)); - this.notificationQueue.push({ - type: "error", - error: s2Error(error), - token, - }); - } - } - })(); - } - - private async sendEntry(entry: InflightEntry): Promise { - if (!this.sessionWriter) { - throw new S2Error({ message: "AppendSession is not ready", status: 500 }); - } - const { precalculatedSize, ...rest } = entry.args ?? {}; - const payload = { - ...(rest as Omit), - records: entry.records, - } as AppendArgs; - - const firstSend = !entry.sent; - entry.sent = true; - if (firstSend) { - entry.enqueuedAt = Date.now(); - } - await this.sessionWriter.write(payload); - if (this.inflight.length > 0 && this.inflight[0] === entry) { - this.startAckTimerForHead(firstSend); - } - } - - private async startRecovery(reason: S2Error): Promise { - debug("Starting recovery due to: %s", reason.message); - if (this.pumpError) return; - if (this.closing && this.inflight.length === 0) { - await this.maybeFinishClose(); - return; - } - // Only retry errors that are retryable by status and policy - if (!isRetryable(reason)) { - this.setPumpError(reason); - return; - } - if ( - (this.retryConfig.appendRetryPolicy ?? "noSideEffects") === - "noSideEffects" && - this.inflight.some((e) => !this.isAppendRetryAllowed(e)) - ) { - // Abort via notification so that backlog and queued requests are rejected too - this.notificationQueue.push({ type: "abort", error: reason }); - return; - } - if (!this.pausedForRecovery) { - this.pausedForRecovery = true; - this.recoveryBarrierLeft = this.inflight.length; - debug( - "pausing enqueues for recovery; barrier=%d inflight=%d backlog=%d", - this.recoveryBarrierLeft, - this.inflight.length, - this.backlog.length, - ); - } - if (!this.recovering) { - this.recovering = true; - void this.runRecoveryLoop(); - } else { - this.restartRequested = true; - } - } - - private async runRecoveryLoop(): Promise { - while (this.recovering) { - this.consecutiveFailures += 1; - debug("Recovering RetryAppendSession (attempt %d)", this.consecutiveFailures); - if (this.pumpStopped || this.closing) return; - - await this.teardownSession(true); - if (this.consecutiveFailures > this.retryConfig.maxAttempts) { - const error = new S2Error({ - message: `Max retry attempts (${this.retryConfig.maxAttempts}) exceeded`, - status: 500, - }); - this.failAll(error); - this.stopPump(error); - return; - } - - const delay = calculateDelay( - this.consecutiveFailures - 1, - this.retryConfig.retryBackoffDurationMs, - ); - await sleep(delay); - if (this.pumpStopped || this.closing) return; - - try { - this.session = await this.generator(this.sessionOptions); - this.sessionWriter = this.session.writable.getWriter(); - this.sessionReader = this.session.acks().getReader(); - this.startAckReader(this.sessionReader); - } catch (e) { - this.restartRequested = true; - } - - debug("resending inflight batches (%d)", this.inflight.length); - for (const entry of this.inflight) { - try { - if ( - (this.retryConfig.appendRetryPolicy ?? "noSideEffects") === - "noSideEffects" && - !this.isAppendRetryAllowed(entry) - ) { - // Abort and surface a policy error; do not continue resending. - this.notificationQueue.push({ - type: "abort", - error: new S2Error({ - message: "Append retry not allowed by policy", - status: 500, - }), - }); - this.restartRequested = false; - return; - } - await this.ensureSession(); - await this.sendEntry(entry); - } catch (error) { - this.restartRequested = true; - break; - } - } - debug("resending complete"); - - if (this.restartRequested) { - this.restartRequested = false; - continue; - } - - // If there are no inflight entries to wait for, flush backlog immediately and resume. - if (this.pausedForRecovery && this.recoveryBarrierLeft === 0) { - debug("recovery barrier is zero; flushing backlog now"); - await this.flushBacklog(); - this.pausedForRecovery = false; - this.recovering = false; - this.consecutiveFailures = 0; - // Wake any blocked writers. - while (this.capacityWaiters.length > 0) { - const resolve = this.capacityWaiters.shift(); - resolve?.(); - } - await this.maybeFinishClose(); - return; - } - - while (this.pausedForRecovery && this.recoveryBarrierLeft > 0 && !this.restartRequested && !this.pumpError && !this.closing) { - await sleep(5); - } - if (this.restartRequested) { - this.restartRequested = false; - continue; - } - if (!this.pausedForRecovery && this.recoveryBarrierLeft === 0) { - this.recovering = false; - this.consecutiveFailures = 0; - await this.maybeFinishClose(); - return; - } - } - } - - // performRecovery and sendUnsentInflightEntries removed in favor of explicit runRecoveryLoop + gating - - private async teardownSession(preserveAckTimer?: boolean): Promise { - if (!preserveAckTimer) { - this.clearAckTimeout(); - } - if (this.sessionReader) { - try { - if (this.readerTokenId !== undefined) { - debug("teardown: cancel reader[%d]", this.readerTokenId); - } - await this.sessionReader.cancel(); - } catch (error) { - debug("Error cancelling ack reader: %s", error); - } - } - this.sessionReader = undefined; - this.readerToken = undefined; - this.readerTokenId = undefined; - - if (this.sessionWriter) { - try { - this.sessionWriter.releaseLock(); - } catch (error) { - debug("Error releasing session writer: %s", error); - } - } - this.sessionWriter = undefined; - - if (this.session) { - try { - await this.session.close(); - } catch (error) { - debug("Error closing underlying session: %s", error); - } - } - this.session = undefined; - } - - private startAckTimerForHead(force: boolean = false): void { - const head = this.inflight[0]; - if (!head) { - // No inflight; clear any existing timer - this.clearAckTimeout(); - this.ackTimeoutHead = undefined; - return; - } - // If the timer is already armed for this head, do nothing to avoid - // cancel/reschedule loops that could postpone the callback under load. - if (!force && this.ackTimeoutHandle && this.ackTimeoutHead === head) { - return; - } - this.clearAckTimeout(); - const delay = Math.max(0, head.enqueuedAt + this.ackTimeoutMs - Date.now()); - debug( - "arm head timer: force=%s delayMs=%d inflight=%d", - force, - delay, - this.inflight.length, - ); - const token = {}; - this.ackTimeoutHead = head; - this.ackTimeoutToken = token; - this.ackTimeoutHandle = setTimeout(() => { - if (this.ackTimeoutToken !== token) { - return; - } - this.ackTimeoutHead = undefined; - // Ack timeout should abort the session rather than be retried. - this.notificationQueue.push({ - type: "abort", - error: new S2Error({ - message: - "Ack timeout: no acknowledgement received within timeout period", - status: 408, - }), - }); - }, delay); - } - - private clearAckTimeout(): void { - if (this.ackTimeoutHandle) { - clearTimeout(this.ackTimeoutHandle); - this.ackTimeoutHandle = undefined; - this.ackTimeoutToken = undefined; - this.ackTimeoutHead = undefined; - } - } - - private async flushBacklog(): Promise { - if (this.backlog.length === 0) return; - debug("flushing backlog (%d)", this.backlog.length); - const backlog = this.backlog; - this.backlog = []; - for (const entry of backlog) { - try { - this.inflight.push(entry); - await this.ensureSession(); - await this.sendEntry(entry); - } catch (error) { - // On send failure, push error to restart recovery and requeue remaining entries - const idx = backlog.indexOf(entry); - if (idx >= 0 && idx + 1 < backlog.length) { - const remaining = backlog.slice(idx + 1); - // Put back remaining at front so they are not lost - this.backlog.unshift(...remaining); - } - this.notificationQueue.push({ type: "error", error: s2Error(error) }); - return; - } - } - } - - private async drainInflight(): Promise { - if (this.inflight.length === 0) { - return; - } - const timeoutMs = 30000; - const start = Date.now(); - while (this.inflight.length > 0) { - if (Date.now() - start > timeoutMs) { - throw new S2Error({ - message: "Close timeout: pending acks not received", - status: 408, - }); - } - await sleep(50); - } - } - - private failAll(error: S2Error): void { - for (const entry of this.inflight) { - entry.reject(error); - } - this.inflight.length = 0; - this.queuedBytes = 0; - this.pendingBytes = 0; - this.releaseCapacity(); - } - - private setPumpError(error: S2Error): void { - if (this.pumpError) { - return; - } - this.pumpError = error; - this.failAll(error); - // Also reject any backlogged entries and queued-but-not-processed batches - if (this.backlog.length > 0) { - for (const entry of this.backlog) { - try { - entry.reject(error); - } catch {} - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - } - this.backlog = []; - } - const drained = this.notificationQueue.drain(); - for (const n of drained) { - if ((n as any).type === "batch") { - const bn = n as any; - this.pendingBytes = Math.max(0, this.pendingBytes - bn.entry.meteredBytes); - try { - bn.entry.reject(error); - } catch {} - } - } - if (this.acksController) { - try { - this.acksController.error(error); - } catch (err) { - debug("Error signaling acks controller: %s", err); - } - } - this.stopPump(error); - } - - private releaseCapacity(): void { - // Wake a single waiter; the waiter will re-check capacity under either - // byte-based or batch-based gating. - const resolve = this.capacityWaiters.shift(); - resolve?.(); - } - - private async finishClose(): Promise { - if (this.pumpStopped) { - return; - } - try { - debug("finishClose: rejecting inflight=%d backlog=%d", this.inflight.length, this.backlog.length); - // Reject any backlogged entries that were never sent - if (this.backlog.length > 0) { - for (const entry of this.backlog) { - try { - entry.reject(new S2Error({ message: "AppendSession is closed", status: 400 })); - } catch {} - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - } - this.backlog = []; - } - // Drain and reject any queued-but-not-processed batches. - const drained = this.notificationQueue.drain(); - let drainedBatches = 0; - for (const n of drained) { - if (n.type === "batch") { - drainedBatches += 1; - // Adjust pendingBytes since handleBatch won't run for these entries anymore. - this.pendingBytes = Math.max( - 0, - this.pendingBytes - n.entry.meteredBytes, - ); - try { - n.entry.reject( - new S2Error({ message: "AppendSession is closed", status: 400 }), - ); - } catch {} - } - } - if (this.maxInflightBatches && this.maxInflightBatches > 0 && drainedBatches > 0) { - this.reservedBatches = Math.max(0, this.reservedBatches - drainedBatches); - } - - await this.teardownSession(); - if (!this.pumpError && this.acksController) { - try { - this.acksController.close(); - } catch (error) { - debug("Error closing acks controller: %s", error); - } - } - debug("finishClose: stopping pump"); - // Wake all capacity waiters to unblock any writers waiting on backpressure. - while (this.capacityWaiters.length > 0) { - const resolve = this.capacityWaiters.shift(); - try { - resolve?.(); - } catch {} - } - this.stopPump(); - } catch (error) { - this.stopPump(s2Error(error)); - } - } - - private stopPump(error?: S2Error): void { - if (this.pumpStopped) { - return; - } - this.pumpStopped = true; - if (error && !this.pumpError) { - this.pumpError = error; - } - if (error) { - this.closeReject?.(error); - } else { - this.closeResolve?.(); - } - this.closeResolve = undefined; - this.closeReject = undefined; - this.notificationQueue.push({ type: "stop" }); - } - - private resolveClosePromise(): void { - if (this.closePromise && !this.pumpError) { - this.closeResolve?.(); - } - this.closeResolve = undefined; - this.closeReject = undefined; - } - - private async maybeFinishClose(): Promise { - if (!this.closing || this.pumpStopped) { - return; - } - if (this.inflight.length > 0 || this.pausedForRecovery || this.recovering || this.backlog.length > 0) { - return; - } - await this.finishClose(); - } - - private async waitForCapacity(bytes: number): Promise { - while (!this.closing && !this.pumpError) { - if (this.pausedForRecovery) { - debug( - "waitForCapacity: paused=true (inflight=%d backlog=%d); writer blocked", - this.inflight.length, - this.backlog.length, - ); - await new Promise((resolve) => { - this.capacityWaiters.push(resolve); - }); - continue; - } - if (this.maxInflightBatches && this.maxInflightBatches > 0) { - // Batch-based gating: reserve a slot if available - if (this.inflight.length + this.backlog.length + this.reservedBatches < this.maxInflightBatches) { - this.reservedBatches += 1; - return; - } - } else { - // Byte-based gating - if (this.queuedBytes + this.pendingBytes + bytes <= this.maxQueuedBytes) { - this.pendingBytes += bytes; - return; - } - } - await new Promise((resolve) => { - this.capacityWaiters.push(resolve); - }); - } - if (this.pumpError) { - throw this.pumpError; - } - throw new S2Error({ message: "AppendSession is closed", status: 400 }); - } - - private cloneRecords(records: AppendRecord[]): AppendRecord[] { - return records.map((record) => { - const cloned: AppendRecord = { ...record }; - if (record.body instanceof Uint8Array) { - cloned.body = record.body.slice(); - } - if (record.headers) { - if (Array.isArray(record.headers)) { - cloned.headers = record.headers.map((h) => [ - ...h, - ]) as AppendRecord["headers"]; - } else { - cloned.headers = { ...record.headers } as AppendRecord["headers"]; - } - } - return cloned; - }); - } - - private removeInflightEntry(entry: InflightEntry): void { - const idx = this.inflight.indexOf(entry); - if (idx !== -1) { - this.inflight.splice(idx, 1); - } - this.queuedBytes = Math.max(0, this.queuedBytes - entry.meteredBytes); - this.releaseCapacity(); - } - - private isAppendRetryAllowed(entry: InflightEntry): boolean { - if (this.retryConfig.appendRetryPolicy === "all") { - return true; - } - return !!entry.args?.match_seq_num || !!entry.args?.fencing_token; - } - - private async enqueueBatch( - records: AppendRecord[], - args?: Omit & { precalculatedSize?: number }, - ): Promise<{ - ackPromise: Promise; - enqueuedPromise: Promise; - }> { - if (this.closing || this.pumpStopped) { - throw new S2Error({ message: "AppendSession is closed", status: 400 }); - } - if (this.pumpError) { - throw this.pumpError; - } - - const clonedRecords = this.cloneRecords(records); - const expectedCount = clonedRecords.length; - const meteredBytes = - args?.precalculatedSize ?? - clonedRecords.reduce((sum, record) => sum + meteredSizeBytes(record), 0); - - let resolveAck!: (ack: AppendAck) => void; - let rejectAck!: (err: S2Error) => void; - const ackPromise = new Promise((resolve, reject) => { - resolveAck = resolve; - rejectAck = reject; - }); - - let resolveEnqueued!: () => void; - let rejectEnqueued!: (err: S2Error) => void; - const enqueuedPromise = new Promise((resolve, reject) => { - resolveEnqueued = resolve; - rejectEnqueued = reject; - }); - - try { - await this.waitForCapacity(meteredBytes); - } catch (error) { - const err = s2Error(error); - rejectAck(err); - rejectEnqueued(err); - throw err; - } - - const entry: InflightEntry = { - records: clonedRecords, - args, - expectedCount, - meteredBytes, - enqueuedAt: Date.now(), - sent: false, - resolve: (ack) => resolveAck(ack), - reject: (err) => rejectAck(err), - }; - - this.notificationQueue.push({ type: "batch", entry }); - this.ensurePump(); - resolveEnqueued(); - return { ackPromise, enqueuedPromise }; - } - - async submit( - records: AppendRecord | AppendRecord[], - args?: Omit & { precalculatedSize?: number }, - ): Promise { - const batch = Array.isArray(records) ? records : [records]; - const { ackPromise } = await this.enqueueBatch(batch, args); - return ackPromise; - } - - acks(): AcksStream { - return this.readable as AcksStream; - } - - lastAckedPosition(): AppendAck | undefined { - return this._lastAckedPosition; - } - - async close(): Promise { - if (this.pumpStopped) { - return; - } - if (this.closePromise) { - return this.closePromise; - } - this.closing = true; - this.closePromise = new Promise((resolve, reject) => { - this.closeResolve = resolve; - this.closeReject = reject; - }); - this.ensurePump(); - this.notificationQueue.push({ type: "close" }); - return this.closePromise; - } - - async abort(reason?: unknown): Promise { - const error = - reason instanceof S2Error - ? reason - : new S2Error({ - message: reason?.toString() ?? "AppendSession aborted", - status: 499, - }); - this.closed = true; - this.notificationQueue.push({ type: "abort", error }); - this.ensurePump(); - } - - async [Symbol.asyncDispose](): Promise { - await this.close(); - } -} diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 87f8a6c..a5b86dc 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -125,22 +125,13 @@ export class S2STransport implements SessionTransport { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - // return S2SReadSession.create( - // this.transportConfig.baseUrl, - // this.transportConfig.accessToken, - // stream, - // args, - // options, - // () => this.getConnection(), - // this.transportConfig.basinName, - // ); return RetryReadSession.create( - (a) => { + (myArgs) => { return S2SReadSession.create( this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, - a, + myArgs, options, () => this.getConnection(), this.transportConfig.basinName, From 74da1e48bfa366a92a2dcd007cf692edf6c15f80 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 21:39:04 -0800 Subject: [PATCH 06/26] a --- src/lib/stream/transport/fetch/index.ts | 8 ++++---- src/lib/stream/transport/s2s/index.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 972ef3f..5707e1b 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -555,11 +555,11 @@ export class FetchTransport implements SessionTransport { maxInflightBatches: 1, } as AppendSessionOptions; return RetryAppendSession.create( - (o) => { + (myOptions) => { return FetchAppendSession.create( stream, this.transportConfig, - o, + myOptions, requestOptions, ); }, @@ -574,8 +574,8 @@ export class FetchTransport implements SessionTransport { options?: S2RequestOptions, ): Promise> { return RetryReadSession.create( - (a) => { - return FetchReadSession.create(this.client, stream, a, options); + (myArgs) => { + return FetchReadSession.create(this.client, stream, myArgs, options); }, args, this.transportConfig.retry, diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index a5b86dc..42dd680 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -104,14 +104,14 @@ export class S2STransport implements SessionTransport { requestOptions?: S2RequestOptions, ): Promise { return RetryAppendSession.create( - (opts) => { + (myOptions) => { return S2SAppendSession.create( this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, () => this.getConnection(), this.transportConfig.basinName, - opts, + myOptions, requestOptions, ); }, From 893809b0aae4bb725033023fbaf0211b2c1fbf08 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 21:41:09 -0800 Subject: [PATCH 07/26] a --- examples/image.ts | 16 +- package-lock.json | 3373 ---------------------- src/batch-transform.ts | 4 +- src/common.ts | 8 + src/lib/retry.ts | 61 +- src/lib/stream/factory.ts | 1 - src/lib/stream/transport/fetch/index.ts | 9 +- src/lib/stream/transport/fetch/shared.ts | 5 +- src/lib/stream/transport/s2s/index.ts | 40 +- src/stream.ts | 16 +- src/tests/retryAppendSession.test.ts | 2 +- 11 files changed, 57 insertions(+), 3478 deletions(-) delete mode 100644 package-lock.json diff --git a/examples/image.ts b/examples/image.ts index d2aef15..ea416d0 100644 --- a/examples/image.ts +++ b/examples/image.ts @@ -29,20 +29,8 @@ function rechunkStream( }); } -// const s2 = new S2({ -// accessToken: process.env.S2_ACCESS_TOKEN!, -// }); - const s2 = new S2({ accessToken: process.env.S2_ACCESS_TOKEN!, - baseUrl: `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, - makeBasinBaseUrl: (basinName) => - `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, - retry: { - maxAttempts: 10, - retryBackoffDurationMs: 1000, - appendRetryPolicy: "all", - }, }); const basinName = process.env.S2_BASIN; @@ -78,8 +66,8 @@ let append = await image // Collect records into batches. .pipeThrough( new BatchTransform({ - lingerDurationMillis: 1, - match_seq_num: 0, + lingerDurationMillis: 50, + match_seq_num: startAt.tail.seq_num, }), ) // Write to the S2 stream. diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index d035618..0000000 --- a/package-lock.json +++ /dev/null @@ -1,3373 +0,0 @@ -{ - "name": "@s2-dev/streamstore", - "version": "0.17.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@s2-dev/streamstore", - "version": "0.17.0", - "license": "Apache-2.0", - "dependencies": { - "@protobuf-ts/runtime": "^2.11.1", - "debug": "^4.4.3" - }, - "devDependencies": { - "@arethetypeswrong/cli": "^0.18.2", - "@biomejs/biome": "2.2.5", - "@changesets/cli": "^2.29.7", - "@hey-api/openapi-ts": "^0.86.0", - "@protobuf-ts/plugin": "^2.11.1", - "@types/bun": "^1.3.1", - "@types/debug": "^4.1.12", - "openapi-typescript": "^7.10.1", - "protoc": "^33.0.0", - "typedoc": "^0.28.14", - "vitest": "^4.0.2" - }, - "peerDependencies": { - "typescript": "^5.9.3" - } - }, - "node_modules/@andrewbranch/untar.js": { - "version": "1.0.3", - "dev": true - }, - "node_modules/@arethetypeswrong/cli": { - "version": "0.18.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@arethetypeswrong/core": "0.18.2", - "chalk": "^4.1.2", - "cli-table3": "^0.6.3", - "commander": "^10.0.1", - "marked": "^9.1.2", - "marked-terminal": "^7.1.0", - "semver": "^7.5.4" - }, - "bin": { - "attw": "dist/index.js" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@arethetypeswrong/core": { - "version": "0.18.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@andrewbranch/untar.js": "^1.0.3", - "@loaderkit/resolve": "^1.0.2", - "cjs-module-lexer": "^1.2.3", - "fflate": "^0.8.2", - "lru-cache": "^11.0.1", - "semver": "^7.5.4", - "typescript": "5.6.1-rc", - "validate-npm-package-name": "^5.0.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@arethetypeswrong/core/node_modules/typescript": { - "version": "5.6.1-rc", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.27.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/runtime": { - "version": "7.28.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@biomejs/biome": { - "version": "2.2.5", - "dev": true, - "license": "MIT OR Apache-2.0", - "bin": { - "biome": "bin/biome" - }, - "engines": { - "node": ">=14.21.3" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/biome" - }, - "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "2.2.5", - "@biomejs/cli-darwin-x64": "2.2.5", - "@biomejs/cli-linux-arm64": "2.2.5", - "@biomejs/cli-linux-arm64-musl": "2.2.5", - "@biomejs/cli-linux-x64": "2.2.5", - "@biomejs/cli-linux-x64-musl": "2.2.5", - "@biomejs/cli-win32-arm64": "2.2.5", - "@biomejs/cli-win32-x64": "2.2.5" - } - }, - "node_modules/@biomejs/cli-darwin-arm64": { - "version": "2.2.5", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@braidai/lang": { - "version": "1.1.2", - "dev": true, - "license": "ISC" - }, - "node_modules/@bufbuild/protobuf": { - "version": "2.10.0", - "dev": true, - "license": "(Apache-2.0 AND BSD-3-Clause)" - }, - "node_modules/@bufbuild/protoplugin": { - "version": "2.10.0", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "2.10.0", - "@typescript/vfs": "^1.5.2", - "typescript": "5.4.5" - } - }, - "node_modules/@bufbuild/protoplugin/node_modules/typescript": { - "version": "5.4.5", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/@changesets/apply-release-plan": { - "version": "7.0.13", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/config": "^3.1.1", - "@changesets/get-version-range-type": "^0.4.0", - "@changesets/git": "^3.0.4", - "@changesets/should-skip-package": "^0.1.2", - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3", - "detect-indent": "^6.0.0", - "fs-extra": "^7.0.1", - "lodash.startcase": "^4.4.0", - "outdent": "^0.5.0", - "prettier": "^2.7.1", - "resolve-from": "^5.0.0", - "semver": "^7.5.3" - } - }, - "node_modules/@changesets/assemble-release-plan": { - "version": "6.0.9", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/errors": "^0.2.0", - "@changesets/get-dependents-graph": "^2.1.3", - "@changesets/should-skip-package": "^0.1.2", - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3", - "semver": "^7.5.3" - } - }, - "node_modules/@changesets/changelog-git": { - "version": "0.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/types": "^6.1.0" - } - }, - "node_modules/@changesets/cli": { - "version": "2.29.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/apply-release-plan": "^7.0.13", - "@changesets/assemble-release-plan": "^6.0.9", - "@changesets/changelog-git": "^0.2.1", - "@changesets/config": "^3.1.1", - "@changesets/errors": "^0.2.0", - "@changesets/get-dependents-graph": "^2.1.3", - "@changesets/get-release-plan": "^4.0.13", - "@changesets/git": "^3.0.4", - "@changesets/logger": "^0.1.1", - "@changesets/pre": "^2.0.2", - "@changesets/read": "^0.6.5", - "@changesets/should-skip-package": "^0.1.2", - "@changesets/types": "^6.1.0", - "@changesets/write": "^0.4.0", - "@inquirer/external-editor": "^1.0.0", - "@manypkg/get-packages": "^1.1.3", - "ansi-colors": "^4.1.3", - "ci-info": "^3.7.0", - "enquirer": "^2.4.1", - "fs-extra": "^7.0.1", - "mri": "^1.2.0", - "p-limit": "^2.2.0", - "package-manager-detector": "^0.2.0", - "picocolors": "^1.1.0", - "resolve-from": "^5.0.0", - "semver": "^7.5.3", - "spawndamnit": "^3.0.1", - "term-size": "^2.1.0" - }, - "bin": { - "changeset": "bin.js" - } - }, - "node_modules/@changesets/config": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/errors": "^0.2.0", - "@changesets/get-dependents-graph": "^2.1.3", - "@changesets/logger": "^0.1.1", - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3", - "fs-extra": "^7.0.1", - "micromatch": "^4.0.8" - } - }, - "node_modules/@changesets/errors": { - "version": "0.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "extendable-error": "^0.1.5" - } - }, - "node_modules/@changesets/get-dependents-graph": { - "version": "2.1.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3", - "picocolors": "^1.1.0", - "semver": "^7.5.3" - } - }, - "node_modules/@changesets/get-release-plan": { - "version": "4.0.13", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/assemble-release-plan": "^6.0.9", - "@changesets/config": "^3.1.1", - "@changesets/pre": "^2.0.2", - "@changesets/read": "^0.6.5", - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3" - } - }, - "node_modules/@changesets/get-version-range-type": { - "version": "0.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/@changesets/git": { - "version": "3.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/errors": "^0.2.0", - "@manypkg/get-packages": "^1.1.3", - "is-subdir": "^1.1.1", - "micromatch": "^4.0.8", - "spawndamnit": "^3.0.1" - } - }, - "node_modules/@changesets/logger": { - "version": "0.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "picocolors": "^1.1.0" - } - }, - "node_modules/@changesets/parse": { - "version": "0.4.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/types": "^6.1.0", - "js-yaml": "^3.13.1" - } - }, - "node_modules/@changesets/parse/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@changesets/parse/node_modules/js-yaml/node_modules/argparse": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@changesets/pre": { - "version": "2.0.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/errors": "^0.2.0", - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3", - "fs-extra": "^7.0.1" - } - }, - "node_modules/@changesets/read": { - "version": "0.6.5", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/git": "^3.0.4", - "@changesets/logger": "^0.1.1", - "@changesets/parse": "^0.4.1", - "@changesets/types": "^6.1.0", - "fs-extra": "^7.0.1", - "p-filter": "^2.1.0", - "picocolors": "^1.1.0" - } - }, - "node_modules/@changesets/should-skip-package": { - "version": "0.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/types": "^6.1.0", - "@manypkg/get-packages": "^1.1.3" - } - }, - "node_modules/@changesets/types": { - "version": "6.1.0", - "dev": true, - "license": "MIT" - }, - "node_modules/@changesets/write": { - "version": "0.4.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@changesets/types": "^6.1.0", - "fs-extra": "^7.0.1", - "human-id": "^4.1.1", - "prettier": "^2.7.1" - } - }, - "node_modules/@colors/colors": { - "version": "1.5.0", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.11", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@gerrit0/mini-shiki": { - "version": "3.14.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@shikijs/engine-oniguruma": "^3.14.0", - "@shikijs/langs": "^3.14.0", - "@shikijs/themes": "^3.14.0", - "@shikijs/types": "^3.14.0", - "@shikijs/vscode-textmate": "^10.0.2" - } - }, - "node_modules/@hey-api/codegen-core": { - "version": "0.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.19.0" - }, - "funding": { - "url": "https://github.com/sponsors/hey-api" - }, - "peerDependencies": { - "typescript": ">=5.5.3" - } - }, - "node_modules/@hey-api/json-schema-ref-parser": { - "version": "1.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@jsdevtools/ono": "^7.1.3", - "@types/json-schema": "^7.0.15", - "js-yaml": "^4.1.0", - "lodash": "^4.17.21" - }, - "engines": { - "node": ">= 16" - }, - "funding": { - "url": "https://github.com/sponsors/hey-api" - } - }, - "node_modules/@hey-api/openapi-ts": { - "version": "0.86.8", - "dev": true, - "license": "MIT", - "dependencies": { - "@hey-api/codegen-core": "^0.3.1", - "@hey-api/json-schema-ref-parser": "1.2.1", - "ansi-colors": "4.1.3", - "c12": "3.3.1", - "color-support": "1.1.3", - "commander": "14.0.1", - "handlebars": "4.7.8", - "open": "10.2.0", - "semver": "7.7.2" - }, - "bin": { - "openapi-ts": "bin/run.js" - }, - "engines": { - "node": ">=20.19.0" - }, - "funding": { - "url": "https://github.com/sponsors/hey-api" - }, - "peerDependencies": { - "typescript": ">=5.5.3" - } - }, - "node_modules/@hey-api/openapi-ts/node_modules/commander": { - "version": "14.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20" - } - }, - "node_modules/@hey-api/openapi-ts/node_modules/semver": { - "version": "7.7.2", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@inquirer/external-editor": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "dependencies": { - "chardet": "^2.1.0", - "iconv-lite": "^0.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "dev": true, - "license": "MIT" - }, - "node_modules/@jsdevtools/ono": { - "version": "7.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@loaderkit/resolve": { - "version": "1.0.4", - "dev": true, - "license": "ISC", - "dependencies": { - "@braidai/lang": "^1.0.0" - } - }, - "node_modules/@manypkg/find-root": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.5.5", - "@types/node": "^12.7.1", - "find-up": "^4.1.0", - "fs-extra": "^8.1.0" - } - }, - "node_modules/@manypkg/find-root/node_modules/@types/node": { - "version": "12.20.55", - "dev": true, - "license": "MIT" - }, - "node_modules/@manypkg/find-root/node_modules/fs-extra": { - "version": "8.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/@manypkg/get-packages": { - "version": "1.1.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.5.5", - "@changesets/types": "^4.0.1", - "@manypkg/find-root": "^1.1.0", - "fs-extra": "^8.1.0", - "globby": "^11.0.0", - "read-yaml-file": "^1.1.0" - } - }, - "node_modules/@manypkg/get-packages/node_modules/@changesets/types": { - "version": "4.1.0", - "dev": true, - "license": "MIT" - }, - "node_modules/@manypkg/get-packages/node_modules/fs-extra": { - "version": "8.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@protobuf-ts/plugin": { - "version": "2.11.1", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "^2.4.0", - "@bufbuild/protoplugin": "^2.4.0", - "@protobuf-ts/protoc": "^2.11.1", - "@protobuf-ts/runtime": "^2.11.1", - "@protobuf-ts/runtime-rpc": "^2.11.1", - "typescript": "^3.9" - }, - "bin": { - "protoc-gen-dump": "bin/protoc-gen-dump", - "protoc-gen-ts": "bin/protoc-gen-ts" - } - }, - "node_modules/@protobuf-ts/plugin/node_modules/typescript": { - "version": "3.9.10", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, - "node_modules/@protobuf-ts/protoc": { - "version": "2.11.1", - "dev": true, - "license": "Apache-2.0", - "bin": { - "protoc": "protoc.js" - } - }, - "node_modules/@protobuf-ts/runtime": { - "version": "2.11.1", - "resolved": "https://registry.npmjs.org/@protobuf-ts/runtime/-/runtime-2.11.1.tgz", - "integrity": "sha512-KuDaT1IfHkugM2pyz+FwiY80ejWrkH1pAtOBOZFuR6SXEFTsnb/jiQWQ1rCIrcKx2BtyxnxW6BWwsVSA/Ie+WQ==", - "license": "(Apache-2.0 AND BSD-3-Clause)" - }, - "node_modules/@protobuf-ts/runtime-rpc": { - "version": "2.11.1", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@protobuf-ts/runtime": "^2.11.1" - } - }, - "node_modules/@redocly/ajv": { - "version": "8.11.3", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js-replace": "^1.0.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@redocly/config": { - "version": "0.22.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@redocly/openapi-core": { - "version": "1.34.5", - "dev": true, - "license": "MIT", - "dependencies": { - "@redocly/ajv": "^8.11.2", - "@redocly/config": "^0.22.0", - "colorette": "^1.2.0", - "https-proxy-agent": "^7.0.5", - "js-levenshtein": "^1.1.6", - "js-yaml": "^4.1.0", - "minimatch": "^5.0.1", - "pluralize": "^8.0.0", - "yaml-ast-parser": "0.0.43" - }, - "engines": { - "node": ">=18.17.0", - "npm": ">=9.5.0" - } - }, - "node_modules/@redocly/openapi-core/node_modules/minimatch": { - "version": "5.1.6", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.52.5", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@shikijs/engine-oniguruma": { - "version": "3.14.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@shikijs/types": "3.14.0", - "@shikijs/vscode-textmate": "^10.0.2" - } - }, - "node_modules/@shikijs/langs": { - "version": "3.14.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@shikijs/types": "3.14.0" - } - }, - "node_modules/@shikijs/themes": { - "version": "3.14.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@shikijs/types": "3.14.0" - } - }, - "node_modules/@shikijs/types": { - "version": "3.14.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.4" - } - }, - "node_modules/@shikijs/vscode-textmate": { - "version": "10.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@sindresorhus/is": { - "version": "4.6.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, - "node_modules/@standard-schema/spec": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/bun": { - "version": "1.3.1", - "dev": true, - "license": "MIT", - "dependencies": { - "bun-types": "1.3.1" - } - }, - "node_modules/@types/chai": { - "version": "5.2.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/deep-eql": "*", - "assertion-error": "^2.0.1" - } - }, - "node_modules/@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/ms": "*" - } - }, - "node_modules/@types/deep-eql": { - "version": "4.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.8", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/hast": { - "version": "3.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/ms": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", - "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "24.9.2", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/react": { - "version": "19.2.2", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "csstype": "^3.0.2" - } - }, - "node_modules/@types/unist": { - "version": "3.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@typescript/vfs": { - "version": "1.6.2", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.1" - }, - "peerDependencies": { - "typescript": "*" - } - }, - "node_modules/@vitest/expect": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.0.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.0.4", - "@vitest/utils": "4.0.4", - "chai": "^6.0.1", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.0.4", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.19" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0-0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.0.4", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/snapshot": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.0.4", - "magic-string": "^0.30.19", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/spy": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.0.4", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ansi-colors": { - "version": "4.1.3", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/ansi-escapes": { - "version": "7.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "environment": "^1.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/any-promise": { - "version": "1.3.0", - "dev": true, - "license": "MIT" - }, - "node_modules/argparse": { - "version": "2.0.1", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/array-union": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/assertion-error": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/better-path-resolve": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "is-windows": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/bun-types": { - "version": "1.3.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - }, - "peerDependencies": { - "@types/react": "^19" - } - }, - "node_modules/bundle-name": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "run-applescript": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/c12": { - "version": "3.3.1", - "dev": true, - "license": "MIT", - "dependencies": { - "chokidar": "^4.0.3", - "confbox": "^0.2.2", - "defu": "^6.1.4", - "dotenv": "^17.2.3", - "exsolve": "^1.0.7", - "giget": "^2.0.0", - "jiti": "^2.6.1", - "ohash": "^2.0.11", - "pathe": "^2.0.3", - "perfect-debounce": "^2.0.0", - "pkg-types": "^2.3.0", - "rc9": "^2.1.2" - }, - "peerDependencies": { - "magicast": "^0.3.5" - }, - "peerDependenciesMeta": { - "magicast": { - "optional": true - } - } - }, - "node_modules/chai": { - "version": "6.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/change-case": { - "version": "5.4.4", - "dev": true, - "license": "MIT" - }, - "node_modules/char-regex": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/chardet": { - "version": "2.1.0", - "dev": true, - "license": "MIT" - }, - "node_modules/chokidar": { - "version": "4.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/ci-info": { - "version": "3.9.0", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/citty": { - "version": "0.1.6", - "dev": true, - "license": "MIT", - "dependencies": { - "consola": "^3.2.3" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "dev": true, - "license": "MIT" - }, - "node_modules/cli-highlight": { - "version": "2.1.11", - "dev": true, - "license": "ISC", - "dependencies": { - "chalk": "^4.0.0", - "highlight.js": "^10.7.1", - "mz": "^2.4.0", - "parse5": "^5.1.1", - "parse5-htmlparser2-tree-adapter": "^6.0.0", - "yargs": "^16.0.0" - }, - "bin": { - "highlight": "bin/highlight" - }, - "engines": { - "node": ">=8.0.0", - "npm": ">=5.0.0" - } - }, - "node_modules/cli-table3": { - "version": "0.6.5", - "dev": true, - "license": "MIT", - "dependencies": { - "string-width": "^4.2.0" - }, - "engines": { - "node": "10.* || >= 12.*" - }, - "optionalDependencies": { - "@colors/colors": "1.5.0" - } - }, - "node_modules/cliui": { - "version": "7.0.4", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "dev": true, - "license": "MIT" - }, - "node_modules/color-support": { - "version": "1.1.3", - "dev": true, - "license": "ISC", - "bin": { - "color-support": "bin.js" - } - }, - "node_modules/colorette": { - "version": "1.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/commander": { - "version": "10.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14" - } - }, - "node_modules/confbox": { - "version": "0.2.2", - "dev": true, - "license": "MIT" - }, - "node_modules/consola": { - "version": "3.4.2", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.18.0 || >=16.10.0" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/csstype": { - "version": "3.1.3", - "dev": true, - "license": "MIT", - "peer": true - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/default-browser": { - "version": "5.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "bundle-name": "^4.1.0", - "default-browser-id": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/defu": { - "version": "6.1.4", - "dev": true, - "license": "MIT" - }, - "node_modules/destr": { - "version": "2.0.5", - "dev": true, - "license": "MIT" - }, - "node_modules/detect-indent": { - "version": "6.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/emojilib": { - "version": "2.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/enquirer": { - "version": "2.4.1", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-colors": "^4.1.1", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/environment": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/es-module-lexer": { - "version": "1.7.0", - "dev": true, - "license": "MIT" - }, - "node_modules/esbuild": { - "version": "0.25.11", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.11", - "@esbuild/android-arm": "0.25.11", - "@esbuild/android-arm64": "0.25.11", - "@esbuild/android-x64": "0.25.11", - "@esbuild/darwin-arm64": "0.25.11", - "@esbuild/darwin-x64": "0.25.11", - "@esbuild/freebsd-arm64": "0.25.11", - "@esbuild/freebsd-x64": "0.25.11", - "@esbuild/linux-arm": "0.25.11", - "@esbuild/linux-arm64": "0.25.11", - "@esbuild/linux-ia32": "0.25.11", - "@esbuild/linux-loong64": "0.25.11", - "@esbuild/linux-mips64el": "0.25.11", - "@esbuild/linux-ppc64": "0.25.11", - "@esbuild/linux-riscv64": "0.25.11", - "@esbuild/linux-s390x": "0.25.11", - "@esbuild/linux-x64": "0.25.11", - "@esbuild/netbsd-arm64": "0.25.11", - "@esbuild/netbsd-x64": "0.25.11", - "@esbuild/openbsd-arm64": "0.25.11", - "@esbuild/openbsd-x64": "0.25.11", - "@esbuild/openharmony-arm64": "0.25.11", - "@esbuild/sunos-x64": "0.25.11", - "@esbuild/win32-arm64": "0.25.11", - "@esbuild/win32-ia32": "0.25.11", - "@esbuild/win32-x64": "0.25.11" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/estree-walker": { - "version": "3.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/expect-type": { - "version": "1.2.2", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/exsolve": { - "version": "1.0.7", - "dev": true, - "license": "MIT" - }, - "node_modules/extendable-error": { - "version": "0.1.7", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-glob": { - "version": "3.3.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.8" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fastq": { - "version": "1.19.1", - "dev": true, - "license": "ISC", - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/fflate": { - "version": "0.8.2", - "dev": true, - "license": "MIT" - }, - "node_modules/fill-range": { - "version": "7.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fs-extra": { - "version": "7.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/giget": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "citty": "^0.1.6", - "consola": "^3.4.0", - "defu": "^6.1.4", - "node-fetch-native": "^1.6.6", - "nypm": "^0.6.0", - "pathe": "^2.0.3" - }, - "bin": { - "giget": "dist/cli.mjs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "11.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "dev": true, - "license": "ISC" - }, - "node_modules/handlebars": { - "version": "4.7.8", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.5", - "neo-async": "^2.6.2", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/highlight.js": { - "version": "10.7.3", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "dev": true, - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/human-id": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "bin": { - "human-id": "dist/cli.js" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ignore": { - "version": "5.3.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/index-to-position": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-docker": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-subdir": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "better-path-resolve": "1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/is-windows": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-wsl": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "is-inside-container": "^1.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/jiti": { - "version": "2.6.1", - "dev": true, - "license": "MIT", - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, - "node_modules/js-levenshtein": { - "version": "1.1.6", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/jsonfile": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/linkify-it": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "uc.micro": "^2.0.0" - } - }, - "node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lodash": { - "version": "4.17.21", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.startcase": { - "version": "4.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "11.2.2", - "dev": true, - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/lunr": { - "version": "2.3.9", - "dev": true, - "license": "MIT" - }, - "node_modules/magic-string": { - "version": "0.30.21", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/markdown-it": { - "version": "14.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.4.0", - "linkify-it": "^5.0.0", - "mdurl": "^2.0.0", - "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" - }, - "bin": { - "markdown-it": "bin/markdown-it.mjs" - } - }, - "node_modules/marked": { - "version": "9.1.6", - "dev": true, - "license": "MIT", - "bin": { - "marked": "bin/marked.js" - }, - "engines": { - "node": ">= 16" - } - }, - "node_modules/marked-terminal": { - "version": "7.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-escapes": "^7.0.0", - "ansi-regex": "^6.1.0", - "chalk": "^5.4.1", - "cli-highlight": "^2.1.11", - "cli-table3": "^0.6.5", - "node-emoji": "^2.2.0", - "supports-hyperlinks": "^3.1.0" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "marked": ">=1 <16" - } - }, - "node_modules/marked-terminal/node_modules/chalk": { - "version": "5.6.2", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/mdurl": { - "version": "2.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/merge2": { - "version": "1.4.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/micromatch": { - "version": "4.0.8", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/minimatch": { - "version": "9.0.5", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mri": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "license": "MIT" - }, - "node_modules/mz": { - "version": "2.7.0", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, - "node_modules/nanoid": { - "version": "3.3.11", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/neo-async": { - "version": "2.6.2", - "dev": true, - "license": "MIT" - }, - "node_modules/node-emoji": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@sindresorhus/is": "^4.6.0", - "char-regex": "^1.0.2", - "emojilib": "^2.4.0", - "skin-tone": "^2.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/node-fetch-native": { - "version": "1.6.7", - "dev": true, - "license": "MIT" - }, - "node_modules/nypm": { - "version": "0.6.2", - "dev": true, - "license": "MIT", - "dependencies": { - "citty": "^0.1.6", - "consola": "^3.4.2", - "pathe": "^2.0.3", - "pkg-types": "^2.3.0", - "tinyexec": "^1.0.1" - }, - "bin": { - "nypm": "dist/cli.mjs" - }, - "engines": { - "node": "^14.16.0 || >=16.10.0" - } - }, - "node_modules/nypm/node_modules/tinyexec": { - "version": "1.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/object-assign": { - "version": "4.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ohash": { - "version": "2.0.11", - "dev": true, - "license": "MIT" - }, - "node_modules/open": { - "version": "10.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "default-browser": "^5.2.1", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/openapi-typescript": { - "version": "7.10.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@redocly/openapi-core": "^1.34.5", - "ansi-colors": "^4.1.3", - "change-case": "^5.4.4", - "parse-json": "^8.3.0", - "supports-color": "^10.2.2", - "yargs-parser": "^21.1.1" - }, - "bin": { - "openapi-typescript": "bin/cli.js" - }, - "peerDependencies": { - "typescript": "^5.x" - } - }, - "node_modules/outdent": { - "version": "0.5.0", - "dev": true, - "license": "MIT" - }, - "node_modules/p-filter": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-map": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-map": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/package-manager-detector": { - "version": "0.2.11", - "dev": true, - "license": "MIT", - "dependencies": { - "quansync": "^0.2.7" - } - }, - "node_modules/parse-json": { - "version": "8.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.26.2", - "index-to-position": "^1.1.0", - "type-fest": "^4.39.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parse5": { - "version": "5.1.1", - "dev": true, - "license": "MIT" - }, - "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "6.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "parse5": "^6.0.1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { - "version": "6.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/path-exists": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-type": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/pathe": { - "version": "2.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/perfect-debounce": { - "version": "2.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.3", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pify": { - "version": "4.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/pkg-types": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "confbox": "^0.2.2", - "exsolve": "^1.0.7", - "pathe": "^2.0.3" - } - }, - "node_modules/pluralize": { - "version": "8.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postcss": { - "version": "8.5.6", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.11", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/prettier": { - "version": "2.8.8", - "dev": true, - "license": "MIT", - "bin": { - "prettier": "bin-prettier.js" - }, - "engines": { - "node": ">=10.13.0" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/protoc": { - "version": "33.0.0", - "dev": true, - "license": "Apache-2.0", - "bin": { - "protoc": "protoc.cjs" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/punycode.js": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/quansync": { - "version": "0.2.11", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/antfu" - }, - { - "type": "individual", - "url": "https://github.com/sponsors/sxzz" - } - ], - "license": "MIT" - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/rc9": { - "version": "2.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "defu": "^6.1.4", - "destr": "^2.0.3" - } - }, - "node_modules/read-yaml-file": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.1.5", - "js-yaml": "^3.6.1", - "pify": "^4.0.1", - "strip-bom": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/read-yaml-file/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/read-yaml-file/node_modules/js-yaml/node_modules/argparse": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/readdirp": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/reusify": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/rollup": { - "version": "4.52.5", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "1.0.8" - }, - "bin": { - "rollup": "dist/bin/rollup" - }, - "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" - }, - "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.52.5", - "@rollup/rollup-android-arm64": "4.52.5", - "@rollup/rollup-darwin-arm64": "4.52.5", - "@rollup/rollup-darwin-x64": "4.52.5", - "@rollup/rollup-freebsd-arm64": "4.52.5", - "@rollup/rollup-freebsd-x64": "4.52.5", - "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", - "@rollup/rollup-linux-arm-musleabihf": "4.52.5", - "@rollup/rollup-linux-arm64-gnu": "4.52.5", - "@rollup/rollup-linux-arm64-musl": "4.52.5", - "@rollup/rollup-linux-loong64-gnu": "4.52.5", - "@rollup/rollup-linux-ppc64-gnu": "4.52.5", - "@rollup/rollup-linux-riscv64-gnu": "4.52.5", - "@rollup/rollup-linux-riscv64-musl": "4.52.5", - "@rollup/rollup-linux-s390x-gnu": "4.52.5", - "@rollup/rollup-linux-x64-gnu": "4.52.5", - "@rollup/rollup-linux-x64-musl": "4.52.5", - "@rollup/rollup-openharmony-arm64": "4.52.5", - "@rollup/rollup-win32-arm64-msvc": "4.52.5", - "@rollup/rollup-win32-ia32-msvc": "4.52.5", - "@rollup/rollup-win32-x64-gnu": "4.52.5", - "@rollup/rollup-win32-x64-msvc": "4.52.5", - "fsevents": "~2.3.2" - } - }, - "node_modules/run-applescript": { - "version": "7.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.3", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/siginfo": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/skin-tone": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "unicode-emoji-modifier-base": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/slash": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/spawndamnit": { - "version": "3.0.1", - "dev": true, - "license": "SEE LICENSE IN LICENSE", - "dependencies": { - "cross-spawn": "^7.0.5", - "signal-exit": "^4.0.1" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stackback": { - "version": "0.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/std-env": { - "version": "3.10.0", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width": { - "version": "4.2.3", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "5.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/supports-color": { - "version": "10.2.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/supports-hyperlinks": { - "version": "3.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=14.18" - }, - "funding": { - "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" - } - }, - "node_modules/supports-hyperlinks/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/term-size": { - "version": "2.2.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/thenify": { - "version": "3.3.1", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0" - } - }, - "node_modules/thenify-all": { - "version": "1.6.0", - "dev": true, - "license": "MIT", - "dependencies": { - "thenify": ">= 3.1.0 < 4" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/tinybench": { - "version": "2.9.0", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "0.3.2", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyglobby": { - "version": "0.2.15", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.3" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tinyrainbow": { - "version": "3.0.3", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/type-fest": { - "version": "4.41.0", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typedoc": { - "version": "0.28.14", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@gerrit0/mini-shiki": "^3.12.0", - "lunr": "^2.3.9", - "markdown-it": "^14.1.0", - "minimatch": "^9.0.5", - "yaml": "^2.8.1" - }, - "bin": { - "typedoc": "bin/typedoc" - }, - "engines": { - "node": ">= 18", - "pnpm": ">= 10" - }, - "peerDependencies": { - "typescript": "5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x || 5.5.x || 5.6.x || 5.7.x || 5.8.x || 5.9.x" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "license": "Apache-2.0", - "peer": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/uc.micro": { - "version": "2.1.0", - "dev": true, - "license": "MIT" - }, - "node_modules/uglify-js": { - "version": "3.19.3", - "dev": true, - "license": "BSD-2-Clause", - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "dev": true, - "license": "MIT" - }, - "node_modules/unicode-emoji-modifier-base": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/universalify": { - "version": "0.1.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4.0.0" - } - }, - "node_modules/uri-js-replace": { - "version": "1.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/validate-npm-package-name": { - "version": "5.0.1", - "dev": true, - "license": "ISC", - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, - "node_modules/vite": { - "version": "7.1.12", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "^0.25.0", - "fdir": "^6.5.0", - "picomatch": "^4.0.3", - "postcss": "^8.5.6", - "rollup": "^4.43.0", - "tinyglobby": "^0.2.15" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "lightningcss": "^1.21.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vitest": { - "version": "4.0.4", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.0.4", - "@vitest/mocker": "4.0.4", - "@vitest/pretty-format": "4.0.4", - "@vitest/runner": "4.0.4", - "@vitest/snapshot": "4.0.4", - "@vitest/spy": "4.0.4", - "@vitest/utils": "4.0.4", - "debug": "^4.4.3", - "es-module-lexer": "^1.7.0", - "expect-type": "^1.2.2", - "magic-string": "^0.30.19", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^3.9.0", - "tinybench": "^2.9.0", - "tinyexec": "^0.3.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.0.3", - "vite": "^6.0.0 || ^7.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@types/debug": "^4.1.12", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.0.4", - "@vitest/browser-preview": "4.0.4", - "@vitest/browser-webdriverio": "4.0.4", - "@vitest/ui": "4.0.4", - "happy-dom": "*", - "jsdom": "*" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@types/debug": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - } - } - }, - "node_modules/which": { - "version": "2.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/why-is-node-running": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wordwrap": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wsl-utils": { - "version": "0.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "is-wsl": "^3.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yaml": { - "version": "2.8.1", - "dev": true, - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - } - }, - "node_modules/yaml-ast-parser": { - "version": "0.0.43", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/yargs": { - "version": "16.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs/node_modules/yargs-parser": { - "version": "20.2.9", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - } - } -} diff --git a/src/batch-transform.ts b/src/batch-transform.ts index aab7a96..d1c8d80 100644 --- a/src/batch-transform.ts +++ b/src/batch-transform.ts @@ -33,7 +33,7 @@ export type BatchOutput = { * @example * ```typescript * const batcher = new BatchTransform<"string">({ - * lingerDuration: 20, + * lingerDurationMillis: 20, * maxBatchRecords: 100, * maxBatchBytes: 256 * 1024, * match_seq_num: 0 // Optional: auto-increments per batch @@ -160,7 +160,7 @@ export class BatchTransform extends TransformStream { if (match_seq_num !== undefined) { batch.match_seq_num = match_seq_num; } - debug!({ batch }); + debug({ batch }); this.controller.enqueue(batch); } diff --git a/src/common.ts b/src/common.ts index cd34f76..8596895 100644 --- a/src/common.ts +++ b/src/common.ts @@ -29,6 +29,14 @@ export type RetryConfig = { * @default "noSideEffects" */ appendRetryPolicy?: AppendRetryPolicy; + + /** + * Maximum time in milliseconds to wait for an append ack before considering + * the attempt timed out and applying retry logic. + * + * Used by retrying append sessions. When unset, defaults to 5000ms. + */ + requestTimeoutMillis?: number; }; /** diff --git a/src/lib/retry.ts b/src/lib/retry.ts index a4086e5..8659685 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -6,7 +6,6 @@ import { meteredSizeBytes } from "../utils.js"; import type { AppendResult, CloseResult } from "./result.js"; import { err, errClose, ok, okClose } from "./result.js"; import type { - AcksStream, AppendArgs, AppendRecord, AppendSession, @@ -24,12 +23,12 @@ const debug = createDebug("s2:retry"); * Default retry configuration. */ export const DEFAULT_RETRY_CONFIG: Required & { - requestTimeoutMs: number; + requestTimeoutMillis: number; } = { maxAttempts: 3, retryBackoffDurationMs: 100, appendRetryPolicy: "noSideEffects", - requestTimeoutMs: 5000, // 5 seconds + requestTimeoutMillis: 5000, // 5 seconds }; const RETRYABLE_STATUS_CODES = new Set([ @@ -151,6 +150,7 @@ export class RetryReadSession implements ReadSession { private _nextReadPosition: StreamPosition | undefined = undefined; + private _lastObservedTail: StreamPosition | undefined = undefined; private _recordsRead: number = 0; private _bytesRead: number = 0; @@ -189,6 +189,11 @@ export class RetryReadSession while (true) { const { done, value: result } = await reader.read(); + // Update last observed tail if transport exposes it + try { + const tail = session.lastObservedTail?.(); + if (tail) this._lastObservedTail = tail; + } catch {} if (done) { reader.releaseLock(); controller.close(); @@ -296,11 +301,11 @@ export class RetryReadSession } lastObservedTail(): StreamPosition | undefined { - return undefined; + return this._lastObservedTail; } nextReadPosition(): StreamPosition | undefined { - return undefined; + return this._nextReadPosition; } } @@ -335,41 +340,6 @@ export class RetryReadSession * - Ack record count matches batch record count * - Acks arrive within ackTimeoutMs (5s) or session is retried */ -class AsyncQueue { - private values: T[] = []; - private waiters: Array<(value: T) => void> = []; - - push(value: T): void { - const waiter = this.waiters.shift(); - if (waiter) { - waiter(value); - return; - } - this.values.push(value); - } - - async next(): Promise { - if (this.values.length > 0) { - return this.values.shift()!; - } - return new Promise((resolve) => { - this.waiters.push(resolve); - }); - } - - clear(): void { - this.values = []; - this.waiters = []; - } - - // Drain currently buffered values (non-blocking) and clear the buffer. - drain(): T[] { - const out = this.values; - this.values = []; - return out; - } -} - /** * New simplified inflight entry for the pump-based architecture. * Each entry tracks a batch and its promise from the inner transport session. @@ -387,11 +357,11 @@ type InflightEntry = { const DEFAULT_MAX_QUEUED_BYTES = 10 * 1024 * 1024; // 10 MiB default export class RetryAppendSession implements AppendSession, AsyncDisposable { - private readonly requestTimeoutMs: number; + private readonly requestTimeoutMillis: number; private readonly maxQueuedBytes: number; private readonly maxInflightBatches?: number; private readonly retryConfig: Required & { - requestTimeoutMs: number; + requestTimeoutMillis: number; }; private readonly inflight: InflightEntry[] = []; @@ -435,7 +405,7 @@ export class RetryAppendSession implements AppendSession, AsyncDisposable { ...DEFAULT_RETRY_CONFIG, ...config, }; - this.requestTimeoutMs = this.retryConfig.requestTimeoutMs; + this.requestTimeoutMillis = this.retryConfig.requestTimeoutMillis; this.maxQueuedBytes = this.sessionOptions?.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES; this.maxInflightBatches = this.sessionOptions?.maxInflightBatches; @@ -467,6 +437,9 @@ export class RetryAppendSession implements AppendSession, AsyncDisposable { delete (args as any).records; args.precalculatedSize = batchMeteredSize; + // Move reserved bytes to queued bytes accounting before submission + this.pendingBytes = Math.max(0, this.pendingBytes - batchMeteredSize); + // Submit without waiting for ack (writable doesn't need per-batch resolution) const promise = this.submitInternal( recordsArray, @@ -903,7 +876,7 @@ export class RetryAppendSession implements AppendSession, AsyncDisposable { private async waitForHead( head: InflightEntry, ): Promise<{ kind: "settled"; value: AppendResult } | { kind: "timeout" }> { - const deadline = head.enqueuedAt + this.requestTimeoutMs; + const deadline = head.enqueuedAt + this.requestTimeoutMillis; const remaining = Math.max(0, deadline - Date.now()); let timer: any; diff --git a/src/lib/stream/factory.ts b/src/lib/stream/factory.ts index efe9b4a..c604937 100644 --- a/src/lib/stream/factory.ts +++ b/src/lib/stream/factory.ts @@ -13,7 +13,6 @@ import type { SessionTransport, TransportConfig } from "./types.js"; * - Everywhere else: uses FetchTransport (JSON over HTTP/1.1) * * @param config Transport configuration - * @param preferHttp2 Force HTTP/2 or HTTP/1.1 (default: auto-detect) */ export async function createSessionTransport( config: TransportConfig, diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 5707e1b..a4b7e21 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -1,4 +1,3 @@ -import { UnknownFieldHandler } from "@protobuf-ts/runtime"; import type { S2RequestOptions } from "../../../../common.js"; import { RangeNotSatisfiableError, S2Error } from "../../../../error.js"; import { @@ -34,8 +33,6 @@ import type { } from "../../types.js"; import { streamAppend } from "./shared.js"; -import last = UnknownFieldHandler.last; - import createDebug from "debug"; import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; @@ -397,7 +394,7 @@ export class FetchAppendSession { */ submit( records: AppendRecord | AppendRecord[], - args?: { fencing_token?: string; match_seq_num?: number }, + args?: { fencing_token?: string; match_seq_num?: number; precalculatedSize?: number }, precalculatedSize?: number, ): Promise { // Validate closed state @@ -423,7 +420,7 @@ export class FetchAppendSession { } // Validate metered size (use precalculated if provided) - let batchMeteredSize = precalculatedSize ?? 0; + let batchMeteredSize = precalculatedSize ?? args?.precalculatedSize ?? 0; if (batchMeteredSize === 0) { for (const record of recordsArray) { batchMeteredSize += meteredSizeBytes(record); @@ -550,6 +547,8 @@ export class FetchTransport implements SessionTransport { sessionOptions?: AppendSessionOptions, requestOptions?: S2RequestOptions, ): Promise { + // Fetch transport intentionally enforces single-flight submission (HTTP/1.1) + // This ensures only one batch is in-flight at a time, regardless of user setting. const opts = { ...sessionOptions, maxInflightBatches: 1, diff --git a/src/lib/stream/transport/fetch/shared.ts b/src/lib/stream/transport/fetch/shared.ts index 8e79811..4598443 100644 --- a/src/lib/stream/transport/fetch/shared.ts +++ b/src/lib/stream/transport/fetch/shared.ts @@ -86,12 +86,12 @@ export async function streamRead( } else { const res: ReadBatch<"string"> = { ...response.data, - records: response.data.records.map((record) => ({ + records: response.data.records?.map((record) => ({ ...record, headers: record.headers ? Object.fromEntries(record.headers) : undefined, - })), + })) }; return res as ReadBatch; } @@ -134,6 +134,7 @@ export async function streamAppend( const format = computeAppendRecordFormat(record); if (format === "bytes") { const formattedRecord = record as AppendRecordForFormat<"bytes">; + hasAnyBytesRecords = true; const encodedRecord = { ...formattedRecord, body: formattedRecord.body diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 42dd680..d4a1aba 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -8,11 +8,6 @@ import * as http2 from "node:http2"; import createDebug from "debug"; import type { S2RequestOptions } from "../../../../common.js"; -import { - type Client, - createClient, - createConfig, -} from "../../../../generated/client/index.js"; import type { AppendAck, StreamPosition } from "../../../../generated/index.js"; import { AppendAck as ProtoAppendAck, @@ -39,7 +34,6 @@ import type { TransportConfig, TransportReadSession, } from "../../types.js"; -import { FetchReadSession } from "../fetch/index.js"; import { frameMessage, S2SFrameParser } from "./framing.js"; const debug = createDebug("s2:s2s"); @@ -82,19 +76,11 @@ export function buildProtoAppendInput( } export class S2STransport implements SessionTransport { - private readonly client: Client; private readonly transportConfig: TransportConfig; private connection?: http2.ClientHttp2Session; private connectionPromise?: Promise; constructor(config: TransportConfig) { - this.client = createClient( - createConfig({ - baseUrl: config.baseUrl, - auth: () => Redacted.value(config.accessToken), - headers: config.basinName ? { "s2-basin": config.basinName } : {}, - }), - ); this.transportConfig = config; } @@ -445,12 +431,12 @@ class S2SReadSession controller.enqueue({ ok: true, value: converted }); // Update next read position to after this record - if (record.seqNum !== undefined) { - this._nextReadPosition = { - seq_num: Number(record.seqNum) + 1, - timestamp: 0, - }; - } + if (record.seqNum !== undefined) { + this._nextReadPosition = { + seq_num: Number(record.seqNum) + 1, + timestamp: Number(record.timestamp ?? 0n), + }; + } } } catch (err) { safeError( @@ -477,7 +463,7 @@ class S2SReadSession stream.on("end", () => { if (stream.rstCode != 0) { - debug!("stream reset code=%d", stream.rstCode); + debug("stream reset code=%d", stream.rstCode); safeError( new S2Error({ message: `Stream ended with error: ${stream.rstCode}`, @@ -861,7 +847,7 @@ class S2SAppendSession { */ async submit( records: AppendRecord | AppendRecord[], - args?: { fencing_token?: string; match_seq_num?: number }, + args?: { fencing_token?: string; match_seq_num?: number; precalculatedSize?: number }, ): Promise { // Validate closed state if (this.closed) { @@ -898,10 +884,12 @@ class S2SAppendSession { ); } - // Calculate metered size - let batchMeteredSize = 0; - for (const record of recordsArray) { - batchMeteredSize += meteredSizeBytes(record); + // Calculate metered size (use precalculated if provided) + let batchMeteredSize = args?.precalculatedSize ?? 0; + if (batchMeteredSize === 0) { + for (const record of recordsArray) { + batchMeteredSize += meteredSizeBytes(record); + } } if (batchMeteredSize > 1024 * 1024) { diff --git a/src/stream.ts b/src/stream.ts index 5a41127..f0e9818 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,9 +1,8 @@ -import createDebug from "debug"; import type { RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Error } from "./error.js"; +import { withS2Error } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AppendAck, checkTail } from "./generated/index.js"; -import { withRetries } from "./lib/retry.js"; +import { isRetryable, withRetries } from "./lib/retry.js"; import { createSessionTransport } from "./lib/stream/factory.js"; import { streamAppend, @@ -21,8 +20,6 @@ import type { TransportConfig, } from "./lib/stream/types.js"; -const debug = createDebug("s2:stream"); - export class S2Stream { private readonly client: Client; private readonly transportConfig: TransportConfig; @@ -123,11 +120,10 @@ export class S2Stream { }, (config, error) => { if ((config.appendRetryPolicy ?? "noSideEffects") === "noSideEffects") { - return ( - !!args?.match_seq_num || - error.status === 429 || - error.status === 502 - ); + // Allow retry if the append is idempotent (match_seq_num or fencing_token) + // or if the error qualifies as retryable by shared logic. + const isIdempotent = !!args?.match_seq_num || !!args?.fencing_token; + return isIdempotent || isRetryable(error); } else { return true; } diff --git a/src/tests/retryAppendSession.test.ts b/src/tests/retryAppendSession.test.ts index 800c3fc..26f31a8 100644 --- a/src/tests/retryAppendSession.test.ts +++ b/src/tests/retryAppendSession.test.ts @@ -191,7 +191,7 @@ describe("RetryAppendSession (unit)", () => { // Accept writes but never emit acks return new FakeTransportAppendSession({ neverAck: true }); }); - (session as any).requestTimeoutMs = 500; + (session as any).requestTimeoutMillis = 500; const ackP = session.submit([{ body: "x" }]); From cbad8747aa5c7c989d55e805df982e6f59ea5763 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 22:56:53 -0800 Subject: [PATCH 08/26] a --- src/lib/retry.ts | 1 + src/lib/stream/transport/fetch/index.ts | 11 +++++++---- src/lib/stream/transport/fetch/shared.ts | 2 +- src/lib/stream/transport/s2s/index.ts | 18 +++++++++++------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 8659685..b6503a4 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -6,6 +6,7 @@ import { meteredSizeBytes } from "../utils.js"; import type { AppendResult, CloseResult } from "./result.js"; import { err, errClose, ok, okClose } from "./result.js"; import type { + AcksStream, AppendArgs, AppendRecord, AppendSession, diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index a4b7e21..f9c9beb 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -1,3 +1,4 @@ +import createDebug from "debug"; import type { S2RequestOptions } from "../../../../common.js"; import { RangeNotSatisfiableError, S2Error } from "../../../../error.js"; import { @@ -17,6 +18,7 @@ import { EventStream } from "../../../event-stream.js"; import * as Redacted from "../../../redacted.js"; import type { AppendResult, CloseResult } from "../../../result.js"; import { err, errClose, ok, okClose } from "../../../result.js"; +import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; import type { AppendArgs, AppendRecord, @@ -33,9 +35,6 @@ import type { } from "../../types.js"; import { streamAppend } from "./shared.js"; -import createDebug from "debug"; -import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; - const debug = createDebug("s2:fetch"); export class FetchReadSession @@ -394,7 +393,11 @@ export class FetchAppendSession { */ submit( records: AppendRecord | AppendRecord[], - args?: { fencing_token?: string; match_seq_num?: number; precalculatedSize?: number }, + args?: { + fencing_token?: string; + match_seq_num?: number; + precalculatedSize?: number; + }, precalculatedSize?: number, ): Promise { // Validate closed state diff --git a/src/lib/stream/transport/fetch/shared.ts b/src/lib/stream/transport/fetch/shared.ts index 4598443..1744ede 100644 --- a/src/lib/stream/transport/fetch/shared.ts +++ b/src/lib/stream/transport/fetch/shared.ts @@ -91,7 +91,7 @@ export async function streamRead( headers: record.headers ? Object.fromEntries(record.headers) : undefined, - })) + })), }; return res as ReadBatch; } diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index d4a1aba..c778ae6 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -431,12 +431,12 @@ class S2SReadSession controller.enqueue({ ok: true, value: converted }); // Update next read position to after this record - if (record.seqNum !== undefined) { - this._nextReadPosition = { - seq_num: Number(record.seqNum) + 1, - timestamp: Number(record.timestamp ?? 0n), - }; - } + if (record.seqNum !== undefined) { + this._nextReadPosition = { + seq_num: Number(record.seqNum) + 1, + timestamp: Number(record.timestamp ?? 0n), + }; + } } } catch (err) { safeError( @@ -847,7 +847,11 @@ class S2SAppendSession { */ async submit( records: AppendRecord | AppendRecord[], - args?: { fencing_token?: string; match_seq_num?: number; precalculatedSize?: number }, + args?: { + fencing_token?: string; + match_seq_num?: number; + precalculatedSize?: number; + }, ): Promise { // Validate closed state if (this.closed) { From faa9dbb47db5895cfdd6a5c4089b619ba63f972d Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 23:02:50 -0800 Subject: [PATCH 09/26] a --- src/tests/retry.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tests/retry.test.ts b/src/tests/retry.test.ts index cfbf17f..9d33716 100644 --- a/src/tests/retry.test.ts +++ b/src/tests/retry.test.ts @@ -57,10 +57,6 @@ describe("Retry Logic", () => { expect(fn).toHaveBeenCalledTimes(2); }); - // Note: Tests for connection and abort errors will be added after implementing - // error kind discrimination (see claude.md). For now, withS2Error converts these - // to S2Error with status=undefined, and isRetryable doesn't handle them yet. - it("should exhaust retries and throw last error", async () => { const error = new S2Error({ message: "Server error", status: 503 }); const fn = vi.fn().mockRejectedValue(error); From 73f6867d8f99d5de4872b53fc6a495a18d335ea4 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 23:15:30 -0800 Subject: [PATCH 10/26] just call them Append/ReadSession --- src/lib/retry.ts | 19 ++++++++----------- src/lib/stream/transport/fetch/index.ts | 9 ++++++--- src/lib/stream/transport/s2s/index.ts | 9 ++++++--- src/lib/stream/types.ts | 10 +++++----- src/tests/retryAppendSession.test.ts | 22 +++++++++++----------- src/tests/retryReadSession.test.ts | 20 ++++++++++---------- 6 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/lib/retry.ts b/src/lib/retry.ts index b6503a4..8d3f25e 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -9,11 +9,9 @@ import type { AcksStream, AppendArgs, AppendRecord, - AppendSession, AppendSessionOptions, ReadArgs, ReadRecord, - ReadSession, TransportAppendSession, TransportReadSession, } from "./stream/types.js"; @@ -146,10 +144,9 @@ export async function withRetries( throw lastError; } -export class RetryReadSession - extends ReadableStream> - implements ReadSession -{ +export class ReadSession< + Format extends "string" | "bytes" = "string", +> extends ReadableStream> { private _nextReadPosition: StreamPosition | undefined = undefined; private _lastObservedTail: StreamPosition | undefined = undefined; @@ -163,7 +160,7 @@ export class RetryReadSession args: ReadArgs = {}, config?: RetryConfig, ) { - return new RetryReadSession(args, generator, config); + return new ReadSession(args, generator, config); } private constructor( @@ -311,7 +308,7 @@ export class RetryReadSession } /** - * RetryAppendSession wraps an underlying AppendSession with automatic retry logic. + * AppendSession wraps an underlying transport AppendSession with automatic retry logic. * * Architecture: * - All writes (submit() and writable.write()) are serialized through inflightQueue @@ -357,7 +354,7 @@ type InflightEntry = { const DEFAULT_MAX_QUEUED_BYTES = 10 * 1024 * 1024; // 10 MiB default -export class RetryAppendSession implements AppendSession, AsyncDisposable { +export class AppendSession implements AsyncDisposable { private readonly requestTimeoutMillis: number; private readonly maxQueuedBytes: number; private readonly maxInflightBatches?: number; @@ -470,8 +467,8 @@ export class RetryAppendSession implements AppendSession, AsyncDisposable { ) => Promise, sessionOptions?: AppendSessionOptions, config?: RetryConfig, - ): Promise { - return new RetryAppendSession(generator, sessionOptions, config); + ): Promise { + return new AppendSession(generator, sessionOptions, config); } /** diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index f9c9beb..fa23503 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -18,7 +18,10 @@ import { EventStream } from "../../../event-stream.js"; import * as Redacted from "../../../redacted.js"; import type { AppendResult, CloseResult } from "../../../result.js"; import { err, errClose, ok, okClose } from "../../../result.js"; -import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; +import { + AppendSession as AppendSessionImpl, + ReadSession as ReadSessionImpl, +} from "../../../retry.js"; import type { AppendArgs, AppendRecord, @@ -556,7 +559,7 @@ export class FetchTransport implements SessionTransport { ...sessionOptions, maxInflightBatches: 1, } as AppendSessionOptions; - return RetryAppendSession.create( + return AppendSessionImpl.create( (myOptions) => { return FetchAppendSession.create( stream, @@ -575,7 +578,7 @@ export class FetchTransport implements SessionTransport { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - return RetryReadSession.create( + return ReadSessionImpl.create( (myArgs) => { return FetchReadSession.create(this.client, stream, myArgs, options); }, diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index c778ae6..609acf3 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -20,7 +20,10 @@ import { meteredSizeBytes } from "../../../../utils.js"; import * as Redacted from "../../../redacted.js"; import type { AppendResult, CloseResult } from "../../../result.js"; import { err, errClose, ok, okClose } from "../../../result.js"; -import { RetryAppendSession, RetryReadSession } from "../../../retry.js"; +import { + AppendSession as AppendSessionImpl, + ReadSession as ReadSessionImpl, +} from "../../../retry.js"; import type { AppendArgs, AppendRecord, @@ -89,7 +92,7 @@ export class S2STransport implements SessionTransport { sessionOptions?: AppendSessionOptions, requestOptions?: S2RequestOptions, ): Promise { - return RetryAppendSession.create( + return AppendSessionImpl.create( (myOptions) => { return S2SAppendSession.create( this.transportConfig.baseUrl, @@ -111,7 +114,7 @@ export class S2STransport implements SessionTransport { args?: ReadArgs, options?: S2RequestOptions, ): Promise> { - return RetryReadSession.create( + return ReadSessionImpl.create( (myArgs) => { return S2SReadSession.create( this.transportConfig.baseUrl, diff --git a/src/lib/stream/types.ts b/src/lib/stream/types.ts index b9c6748..3ac9236 100644 --- a/src/lib/stream/types.ts +++ b/src/lib/stream/types.ts @@ -66,7 +66,7 @@ export interface AcksStream /** * Transport-facing interface for "dumb" append sessions. * Transports only implement submit/close with value-encoded errors (discriminated unions). - * No backpressure, no retry, no streams - RetryAppendSession adds those. + * No backpressure, no retry, no streams - AppendSession adds those. */ export interface TransportAppendSession { submit( @@ -78,7 +78,7 @@ export interface TransportAppendSession { /** * Public AppendSession interface with retry, backpressure, and streams. - * This is what users interact with - implemented by RetryAppendSession. + * This is what users interact with - implemented by AppendSession. */ export interface AppendSession extends ReadableWritablePair, @@ -108,7 +108,7 @@ export type ReadResult = /** * Transport-level read session interface. * Transport implementations yield ReadResult and never throw errors from the stream. - * RetryReadSession wraps these and converts them to the public ReadSession interface. + * ReadSession wraps these and converts them to the public ReadSession interface. */ export interface TransportReadSession< Format extends "string" | "bytes" = "string", @@ -134,13 +134,13 @@ export interface ReadSession export interface AppendSessionOptions { /** * Maximum bytes to queue before applying backpressure (default: 10 MiB). - * Enforced by RetryAppendSession; underlying transports do not apply + * Enforced by AppendSession; underlying transports do not apply * byte-based backpressure on their own. */ maxQueuedBytes?: number; /** * Maximum number of batches allowed in-flight (including queued) before - * applying backpressure. This is enforced by RetryAppendSession; underlying + * applying backpressure. This is enforced by AppendSession; underlying * transport sessions do not implement their own backpressure. */ maxInflightBatches?: number; diff --git a/src/tests/retryAppendSession.test.ts b/src/tests/retryAppendSession.test.ts index 26f31a8..6528300 100644 --- a/src/tests/retryAppendSession.test.ts +++ b/src/tests/retryAppendSession.test.ts @@ -3,7 +3,7 @@ import { S2Error } from "../error.js"; import type { AppendAck, StreamPosition } from "../generated/index.js"; import type { AppendResult, CloseResult } from "../lib/result.js"; import { err, errClose, ok, okClose } from "../lib/result.js"; -import { RetryAppendSession } from "../lib/retry.js"; +import { AppendSession as AppendSessionImpl } from "../lib/retry.js"; import type { AcksStream, AppendArgs, @@ -13,7 +13,7 @@ import type { } from "../lib/stream/types.js"; /** - * Minimal controllable AppendSession for testing RetryAppendSession. + * Minimal controllable AppendSession for testing AppendSessionImpl. */ class FakeAppendSession implements AppendSession { public readonly readable: ReadableStream; @@ -115,7 +115,7 @@ class FakeAppendSession implements AppendSession { /** * Transport-level fake session that returns discriminated unions. - * Used for testing RetryAppendSession which wraps transport sessions. + * Used for testing AppendSessionImpl which wraps transport sessions. */ class FakeTransportAppendSession implements TransportAppendSession { public writes: Array<{ records: AppendRecord[]; args?: any }> = []; @@ -178,7 +178,7 @@ class FakeTransportAppendSession implements TransportAppendSession { } } -describe("RetryAppendSession (unit)", () => { +describe("AppendSessionImpl (unit)", () => { beforeEach(() => { vi.useFakeTimers(); }); @@ -187,7 +187,7 @@ describe("RetryAppendSession (unit)", () => { }); it("aborts on ack timeout (~5s from enqueue) when no acks arrive", async () => { - const session = await RetryAppendSession.create(async () => { + const session = await AppendSessionImpl.create(async () => { // Accept writes but never emit acks return new FakeTransportAppendSession({ neverAck: true }); }); @@ -212,7 +212,7 @@ describe("RetryAppendSession (unit)", () => { it("recovers from send-phase transient error and resolves after recovery", async () => { // First session rejects writes; second accepts and acks immediately let call = 0; - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => { call++; if (call === 1) { @@ -237,7 +237,7 @@ describe("RetryAppendSession (unit)", () => { it("fails immediately when retries are disabled and send-phase errors persist", async () => { const error = new S2Error({ message: "boom", status: 500 }); - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => new FakeTransportAppendSession({ submitError: error }), undefined, { retryBackoffDurationMs: 1, maxAttempts: 0, appendRetryPolicy: "all" }, @@ -252,7 +252,7 @@ describe("RetryAppendSession (unit)", () => { it("does not retry non-idempotent inflight under noSideEffects policy and exposes failure cause", async () => { const error = new S2Error({ message: "boom", status: 500 }); - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => new FakeTransportAppendSession({ submitError: error }), undefined, { @@ -269,7 +269,7 @@ describe("RetryAppendSession (unit)", () => { it("abort rejects backlog and queued submissions with the abort error", async () => { const error = new S2Error({ message: "boom", status: 500 }); - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => new FakeTransportAppendSession({ submitError: error }), undefined, { @@ -299,7 +299,7 @@ describe("RetryAppendSession (unit)", () => { tail: { seq_num: 1, timestamp: 0 }, }; - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => new FakeTransportAppendSession({ customAcks: [ack1, ack2] }), undefined, { retryBackoffDurationMs: 1, maxAttempts: 0 }, // No retries @@ -346,7 +346,7 @@ describe("RetryAppendSession (unit)", () => { tail: { seq_num: 10, timestamp: 0 }, }; - const session = await RetryAppendSession.create( + const session = await AppendSessionImpl.create( async () => new FakeTransportAppendSession({ customAcks: [ack1, ack2] }), undefined, { retryBackoffDurationMs: 1, maxAttempts: 0 }, diff --git a/src/tests/retryReadSession.test.ts b/src/tests/retryReadSession.test.ts index cca5133..ead1859 100644 --- a/src/tests/retryReadSession.test.ts +++ b/src/tests/retryReadSession.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { S2Error } from "../error.js"; import type { StreamPosition } from "../generated/index.js"; -import { RetryReadSession } from "../lib/retry.js"; +import { ReadSession } from "../lib/retry.js"; import type { ReadArgs, ReadRecord, @@ -10,7 +10,7 @@ import type { } from "../lib/stream/types.js"; /** - * Fake TransportReadSession for testing RetryReadSession. + * Fake TransportReadSession for testing ReadSession. * Implements the transport layer pattern: yields ReadResult and never throws. */ class FakeReadSession @@ -124,7 +124,7 @@ class FakeReadSession } } -describe("RetryReadSession (unit)", () => { +describe("ReadSession (unit)", () => { // Note: Not using fake timers here because they don't play well with async iteration // Instead, we use very short backoff times (1ms) to make tests run fast @@ -138,7 +138,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -182,7 +182,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -226,7 +226,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -272,7 +272,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -315,7 +315,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -358,7 +358,7 @@ describe("RetryReadSession (unit)", () => { let callCount = 0; const capturedArgs: Array> = []; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (args) => { capturedArgs.push({ ...args }); callCount++; @@ -416,7 +416,7 @@ describe("RetryReadSession (unit)", () => { it("fails after max retry attempts exhausted", async () => { let callCount = 0; - const session = await RetryReadSession.create( + const session = await ReadSession.create( async (_args) => { callCount++; // Always error immediately without emitting any successful records From 752af04242b3e0dd548705ce1d0b7dfda39be95f Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 23:17:15 -0800 Subject: [PATCH 11/26] bun lock --- bun.lock | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bun.lock b/bun.lock index d40a77c..90547ea 100644 --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,7 @@ "name": "@s2-dev/streamstore", "dependencies": { "@protobuf-ts/runtime": "^2.11.1", + "debug": "^4.4.3", }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.2", @@ -13,6 +14,7 @@ "@hey-api/openapi-ts": "^0.86.0", "@protobuf-ts/plugin": "^2.11.1", "@types/bun": "^1.3.1", + "@types/debug": "^4.1.12", "openapi-typescript": "^7.10.1", "protoc": "^33.0.0", "typedoc": "^0.28.14", @@ -250,6 +252,8 @@ "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], @@ -258,6 +262,8 @@ "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], + "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], + "@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="], "@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="], From c19002a880672841ac4ea3d24b5d53efa94e7f7a Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Mon, 10 Nov 2025 23:24:12 -0800 Subject: [PATCH 12/26] bun race --- src/lib/retry.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 8d3f25e..81c42c5 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -258,8 +258,15 @@ export class ReadSession< } } }, - cancel: async () => { - session?.cancel(); + cancel: async (reason) => { + try { + await session?.cancel(reason); + } catch (err) { + // Ignore ERR_INVALID_STATE - stream may already be closed/cancelled + if ((err as any)?.code !== "ERR_INVALID_STATE") { + throw err; + } + } }, }); } @@ -283,12 +290,20 @@ export class ReadSession< return { done: false, value: r.value }; }, throw: async (e) => { - await reader.cancel(e); + try { + await reader.cancel(e); + } catch (err) { + if ((err as any)?.code !== "ERR_INVALID_STATE") throw err; + } reader.releaseLock(); return { done: true, value: undefined }; }, return: async () => { - await reader.cancel("done"); + try { + await reader.cancel("done"); + } catch (err) { + if ((err as any)?.code !== "ERR_INVALID_STATE") throw err; + } reader.releaseLock(); return { done: true, value: undefined }; }, From 310cce493d11667dbe79402df019848fe43544e5 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 09:17:01 -0800 Subject: [PATCH 13/26] a --- src/accessTokens.ts | 45 +++++++++++++------------- src/basins.ts | 77 +++++++++++++++++++++------------------------ src/error.ts | 44 +++++++++++++++++++++++--- src/metrics.ts | 49 ++++++++++++++--------------- src/s2.ts | 13 ++++---- src/stream.ts | 19 ++++++----- src/streams.ts | 77 +++++++++++++++++++++------------------------ 7 files changed, 170 insertions(+), 154 deletions(-) diff --git a/src/accessTokens.ts b/src/accessTokens.ts index 995b006..a5f2556 100644 --- a/src/accessTokens.ts +++ b/src/accessTokens.ts @@ -36,14 +36,13 @@ export class S2AccessTokens { */ public async list(args?: ListAccessTokensArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - listAccessTokens({ - client: this.client, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + listAccessTokens({ + client: this.client, + query: args, + ...options, + }), + ); }); return response.data; @@ -59,14 +58,13 @@ export class S2AccessTokens { */ public async issue(args: IssueAccessTokenArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - issueAccessToken({ - client: this.client, - body: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + issueAccessToken({ + client: this.client, + body: args, + ...options, + }), + ); }); return response.data; @@ -79,14 +77,13 @@ export class S2AccessTokens { */ public async revoke(args: RevokeAccessTokenArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - revokeAccessToken({ - client: this.client, - path: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + revokeAccessToken({ + client: this.client, + path: args, + ...options, + }), + ); }); return response.data; diff --git a/src/basins.ts b/src/basins.ts index 5eba10d..10f80c6 100644 --- a/src/basins.ts +++ b/src/basins.ts @@ -47,14 +47,13 @@ export class S2Basins { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - listBasins({ - client: this.client, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + listBasins({ + client: this.client, + query: args, + ...options, + }), + ); }); return response.data; @@ -72,14 +71,13 @@ export class S2Basins { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - createBasin({ - client: this.client, - body: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + createBasin({ + client: this.client, + body: args, + ...options, + }), + ); }); return response.data; @@ -95,14 +93,13 @@ export class S2Basins { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - getBasinConfig({ - client: this.client, - path: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + getBasinConfig({ + client: this.client, + path: args, + ...options, + }), + ); }); return response.data; @@ -118,14 +115,13 @@ export class S2Basins { options?: S2RequestOptions, ): Promise { await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - deleteBasin({ - client: this.client, - path: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + deleteBasin({ + client: this.client, + path: args, + ...options, + }), + ); }); } @@ -140,15 +136,14 @@ export class S2Basins { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - reconfigureBasin({ - client: this.client, - path: args, - body: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + reconfigureBasin({ + client: this.client, + path: args, + body: args, + ...options, + }), + ); }); return response.data; diff --git a/src/error.ts b/src/error.ts index 44e2ca5..bd8de15 100644 --- a/src/error.ts +++ b/src/error.ts @@ -64,11 +64,45 @@ export function s2Error(error: any): S2Error { } export async function withS2Error(fn: () => Promise): Promise { - try { - return await fn(); - } catch (error) { - throw s2Error(error); - } + try { + const result: any = await fn(); + + // Support response-parsing mode (throwOnError=false): + // Generated client responses have shape { data, error?, response } + if ( + result && + typeof result === "object" && + Object.prototype.hasOwnProperty.call(result, "error") + ) { + const err = result.error; + if (err) { + const status = result.response?.status as number | undefined; + const statusText = result.response?.statusText as + | string + | undefined; + + // If server provided structured error with message/code, use it + if (typeof err === "object" && "message" in err) { + throw new S2Error({ + message: (err as any).message ?? statusText ?? "Error", + code: (err as any).code ?? undefined, + status, + }); + } + + // Fallback: synthesize from HTTP response metadata + throw new S2Error({ + message: statusText ?? "Request failed", + status, + }); + } + } + + return result as T; + } catch (error) { + // Network and other thrown errors + throw s2Error(error); + } } /** diff --git a/src/metrics.ts b/src/metrics.ts index 0c32153..5eb0123 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -34,14 +34,13 @@ export class S2Metrics { */ public async account(args: AccountMetricsArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - accountMetrics({ - client: this.client, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + accountMetrics({ + client: this.client, + query: args, + ...options, + }), + ); }); return response.data; @@ -58,15 +57,14 @@ export class S2Metrics { */ public async basin(args: BasinMetricsArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - basinMetrics({ - client: this.client, - path: args, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + basinMetrics({ + client: this.client, + path: args, + query: args, + ...options, + }), + ); }); return response.data; @@ -84,15 +82,14 @@ export class S2Metrics { */ public async stream(args: StreamMetricsArgs, options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - streamMetrics({ - client: this.client, - path: args, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + streamMetrics({ + client: this.client, + path: args, + query: args, + ...options, + }), + ); }); return response.data; diff --git a/src/s2.ts b/src/s2.ts index b436ec1..62e3341 100644 --- a/src/s2.ts +++ b/src/s2.ts @@ -43,13 +43,12 @@ export class S2 { constructor(options: S2ClientOptions) { this.accessToken = Redacted.make(options.accessToken); this.retryConfig = options.retry ?? {}; - this.client = createClient( - createConfig({ - baseUrl: options.baseUrl ?? defaultBaseUrl, - auth: () => Redacted.value(this.accessToken), - throwOnError: true, - }), - ); + this.client = createClient( + createConfig({ + baseUrl: options.baseUrl ?? defaultBaseUrl, + auth: () => Redacted.value(this.accessToken), + }), + ); this.client.interceptors.error.use((err, res, req, opt) => { return new S2Error({ diff --git a/src/stream.ts b/src/stream.ts index f0e9818..4f98b6d 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -57,16 +57,15 @@ export class S2Stream { */ public async checkTail(options?: S2RequestOptions) { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - checkTail({ - client: this.client, - path: { - stream: this.name, - }, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + checkTail({ + client: this.client, + path: { + stream: this.name, + }, + ...options, + }), + ); }); return response.data; diff --git a/src/streams.ts b/src/streams.ts index 0167388..65157d8 100644 --- a/src/streams.ts +++ b/src/streams.ts @@ -48,14 +48,13 @@ export class S2Streams { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - listStreams({ - client: this.client, - query: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + listStreams({ + client: this.client, + query: args, + ...options, + }), + ); }); return response.data; @@ -72,14 +71,13 @@ export class S2Streams { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - createStream({ - client: this.client, - body: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + createStream({ + client: this.client, + body: args, + ...options, + }), + ); }); return response.data; @@ -95,14 +93,13 @@ export class S2Streams { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - getStreamConfig({ - client: this.client, - path: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + getStreamConfig({ + client: this.client, + path: args, + ...options, + }), + ); }); return response.data; @@ -118,14 +115,13 @@ export class S2Streams { options?: S2RequestOptions, ): Promise { await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - deleteStream({ - client: this.client, - path: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + deleteStream({ + client: this.client, + path: args, + ...options, + }), + ); }); } @@ -140,15 +136,14 @@ export class S2Streams { options?: S2RequestOptions, ): Promise { const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => - reconfigureStream({ - client: this.client, - path: args, - body: args, - ...options, - throwOnError: true, - }), - ); + return await withS2Error(async () => + reconfigureStream({ + client: this.client, + path: args, + body: args, + ...options, + }), + ); }); return response.data; From e0a5d93e376ea9f4d5d91222ee46fcfee5fd5fee Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 09:35:38 -0800 Subject: [PATCH 14/26] a --- src/tests/withS2Error.test.ts | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/tests/withS2Error.test.ts diff --git a/src/tests/withS2Error.test.ts b/src/tests/withS2Error.test.ts new file mode 100644 index 0000000..1310bc1 --- /dev/null +++ b/src/tests/withS2Error.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { S2Error, withS2Error } from "../error.js"; + +describe("withS2Error response parsing", () => { + it("returns result when response has no error", async () => { + const value = await withS2Error(async () => ({ + data: { ok: 1 }, + error: undefined, + response: { status: 200, statusText: "OK" }, + })); + + expect(value).toMatchObject({ data: { ok: 1 }, response: { status: 200 } }); + }); + + it("throws S2Error with message/code/status when response.error has message", async () => { + const run = () => + withS2Error(async () => ({ + data: undefined, + error: { message: "Bad things", code: "BAD_THING" }, + response: { status: 400, statusText: "Bad Request" }, + })); + + await expect(run()).rejects.toMatchObject({ + name: "S2Error", + message: "Bad things", + code: "BAD_THING", + status: 400, + }); + }); + + it("falls back to HTTP statusText when error lacks message", async () => { + const run = () => + withS2Error(async () => ({ + data: undefined, + error: { something: "else" }, + response: { status: 502, statusText: "Bad Gateway" }, + })); + + await expect(run()).rejects.toMatchObject({ + name: "S2Error", + message: "Bad Gateway", + status: 502, + }); + }); + + it("wraps thrown errors as S2Error via s2Error()", async () => { + const run = () => + withS2Error(async () => { + throw new Error("boom"); + }); + + const err = await run().catch((e) => e as S2Error); + expect(err).toBeInstanceOf(S2Error); + expect(err.message).toBe("boom"); + // Generic thrown errors get status 0 in s2Error() + expect(err.status).toBe(0); + }); +}); + From f403389a5ae91d7eeac6bccbb6780d1c998385d1 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 15:23:32 -0800 Subject: [PATCH 15/26] read session bug --- examples/image2.ts | 144 +++++++++++++++++++++++++ src/lib/retry.ts | 35 +++--- src/lib/stream/transport/s2s/index.ts | 150 ++++++++++++++++++-------- src/stream.ts | 2 +- src/tests/retryReadSession.test.ts | 52 ++++++++- 5 files changed, 315 insertions(+), 68 deletions(-) create mode 100644 examples/image2.ts diff --git a/examples/image2.ts b/examples/image2.ts new file mode 100644 index 0000000..39ecb1e --- /dev/null +++ b/examples/image2.ts @@ -0,0 +1,144 @@ +import { createWriteStream } from "node:fs"; +import { + AppendRecord, + BatchTransform, + type ReadRecord, + S2, +} from "../src/index.js"; + +function rechunkStream( + desiredChunkSize: number, +): TransformStream { + let buffer = new Uint8Array(0); + return new TransformStream({ + transform(chunk, controller) { + const newBuffer = new Uint8Array(buffer.length + chunk.length); + newBuffer.set(buffer); + newBuffer.set(chunk, buffer.length); + buffer = newBuffer; + while (buffer.length >= desiredChunkSize) { + controller.enqueue(buffer.slice(0, desiredChunkSize)); + buffer = buffer.slice(desiredChunkSize); + } + }, + flush(controller) { + if (buffer.length > 0) { + controller.enqueue(buffer); + } + }, + }); +} + +// const s2 = new S2({ +// accessToken: process.env.S2_ACCESS_TOKEN!, +// }); + +const s2 = new S2({ + accessToken: process.env.S2_ACCESS_TOKEN!, + baseUrl: `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, + makeBasinBaseUrl: (basinName) => + `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, + retry: { + maxAttempts: 10, + retryBackoffDurationMs: 100, + appendRetryPolicy: "all", + requestTimeoutMillis: 20000, + }, +}); + +const basinName = process.env.S2_BASIN; +if (!basinName) { + console.error("S2_BASIN environment variable is not set"); + process.exit(1); +} + +const basin = s2.basin(process.env.S2_BASIN!); +const stream = basin.stream("image"); + +console.log("ct"); +const startAt = await stream.checkTail(); + +console.log(`Tail seqNum=${startAt.tail.seq_num}`); + + +const session = await stream.appendSession({ + maxQueuedBytes: 1024 * 1024 * 1, + maxInflightBatches: 10, +}); +let image = await fetch( + "https://upload.wikimedia.org/wikipedia/commons/2/24/Peter_Paul_Rubens_-_Self-portrait_-_RH.S.180_-_Rubenshuis_%28after_restoration%29.jpg", +); + +// Write directly from fetch response to S2 stream +let append = await image + .body! // Ensure each chunk is at most 128KiB. S2 has a maximum individual record size of 1MiB. + .pipeThrough(rechunkStream(1024 * 128)) + // Convert each chunk to an AppendRecord. + .pipeThrough( + new TransformStream({ + transform(arr, controller) { + controller.enqueue(AppendRecord.make(arr)); + }, + }), + ) + // Collect records into batches. + .pipeThrough( + new BatchTransform({ + lingerDurationMillis: 50, + match_seq_num: startAt.tail.seq_num, + }), + ) + // Write to the S2 stream. + .pipeTo(session.writable); + +console.log( + `image written to S2 over ${session.lastAckedPosition()!.end!.seq_num - startAt.tail.seq_num} records, starting at seqNum=${startAt.tail.seq_num}`, +); + +let lastAcked= session.lastAckedPosition()!.end.seq_num; +console.log("lastAcked=%s", { lastAcked }) + +let readSession = await stream.readSession({ + seq_num: startAt.tail.seq_num, + count: lastAcked - startAt.tail.seq_num, + as: "bytes", +}); + +// Write to a local file. +const id = Math.random().toString(36).slice(2, 10); +// Use a larger buffer (default is 16KB, we use 512KB) +const out = createWriteStream(`image-${id}.jpg`, { + highWaterMark: 512 * 1024, // 512KB buffer +}); + +await readSession + .pipeThrough( + new TransformStream, Uint8Array>({ + transform(arr, controller) { + controller.enqueue(arr.body); + }, + }), + ) + .pipeTo( + new WritableStream({ + async write(chunk) { + // Handle backpressure - wait if buffer is full + if (!out.write(chunk)) { + await new Promise((resolve) => out.once("drain", resolve)); + } + }, + // Don't close here - we'll close manually after to ensure flush + }), + ); + +// Ensure the file is fully written and closed before exiting +await new Promise((resolve, reject) => { + out.close((err) => { + if (err) reject(err); + else resolve(); + }); +}); + +console.log(`Image written to image-${id}.jpg`); + +process.exit(0); diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 81c42c5..39a2036 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -178,10 +178,15 @@ export class ReadSession< const startTimeMs = performance.now(); // Capture start time before super() super({ start: async (controller) => { - let nextArgs = { ...args }; + let nextArgs = { ...args } as ReadArgs; + // Capture original request budget so retries compute from a stable baseline + const baselineCount = args?.count; + const baselineBytes = args?.bytes; + const baselineWait = args?.wait; let attempt = 0; while (true) { + debug("starting read session with args: %o", nextArgs); session = await generator(nextArgs); const reader = session.getReader(); @@ -208,25 +213,19 @@ export class ReadSession< if (this._nextReadPosition) { nextArgs.seq_num = this._nextReadPosition.seq_num; } - if (nextArgs.count) { - nextArgs.count = - this._recordsRead === undefined - ? nextArgs.count - : nextArgs.count - this._recordsRead; + // Recompute remaining budget from original request each time to avoid double-subtraction + if (baselineCount !== undefined) { + const remaining = Math.max(0, baselineCount - this._recordsRead); + nextArgs.count = remaining as any; } - if (nextArgs.bytes) { - nextArgs.bytes = - this._bytesRead === undefined - ? nextArgs.bytes - : nextArgs.bytes - this._bytesRead; + if (baselineBytes !== undefined) { + const remaining = Math.max(0, baselineBytes - this._bytesRead); + nextArgs.bytes = remaining as any; } - // Adjust wait to account for elapsed time. - // If user specified wait=10s and we've already spent 5s (including backoff), - // we should only wait another 5s on retry to honor the original time budget. - if (nextArgs.wait !== undefined) { - const elapsedSeconds = - (performance.now() - startTimeMs) / 1000; - nextArgs.wait = Math.max(0, nextArgs.wait - elapsedSeconds); + // Adjust wait from original budget based on total elapsed time since start + if (baselineWait !== undefined) { + const elapsedSeconds = (performance.now() - startTimeMs) / 1000; + nextArgs.wait = Math.max(0, baselineWait - elapsedSeconds) as any; } const delay = calculateDelay( attempt, diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 609acf3..b8ee4ef 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -15,7 +15,12 @@ import { ReadBatch as ProtoReadBatch, type StreamPosition as ProtoStreamPosition, } from "../../../../generated/proto/s2.js"; -import { S2Error } from "../../../../index.js"; +import { + FencingTokenMismatchError, + RangeNotSatisfiableError, + S2Error, + SeqNumMismatchError, +} from "../../../../error.js"; import { meteredSizeBytes } from "../../../../utils.js"; import * as Redacted from "../../../redacted.js"; import type { AppendResult, CloseResult } from "../../../result.js"; @@ -384,31 +389,42 @@ class S2SReadSession let frame = parser.parseFrame(); while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); - try { - const errorJson = JSON.parse(errorText); - safeError( - new S2Error({ - message: errorJson.message ?? "Unknown error", - code: errorJson.code, - status: frame.statusCode, - }), - ); - } catch { - safeError( - new S2Error({ - message: errorText || "Unknown error", - status: frame.statusCode, - }), - ); - } - } else { - safeClose(); - } - stream.close(); - } else { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + try { + const errorJson = JSON.parse(errorText); + const status = frame.statusCode ?? 500; + const message = errorJson.message ?? "Unknown error"; + const code = errorJson.code; + + // Map known read errors + if (status === 416) { + safeError( + new RangeNotSatisfiableError({ status }), + ); + } else { + safeError( + new S2Error({ + message, + code, + status, + }), + ); + } + } catch { + safeError( + new S2Error({ + message: errorText || "Unknown error", + status: frame.statusCode, + }), + ); + } + } else { + safeClose(); + } + stream.close(); + } else { // Parse ReadBatch try { const protoBatch = ProtoReadBatch.fromBinary(frame.body); @@ -692,26 +708,68 @@ class S2SAppendSession { let frame = this.parser.parseFrame(); while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); - const status = frame.statusCode ?? 500; - try { - const errorJson = JSON.parse(errorText); - const message = errorJson.message ?? "Unknown error"; - const code = errorJson.code; - queueMicrotask(() => - safeError(new S2Error({ message, code, status })), - ); - } catch { - const message = errorText || "Unknown error"; - queueMicrotask(() => - safeError(new S2Error({ message, status })), - ); - } - } - stream.close(); - } else { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + const status = frame.statusCode ?? 500; + try { + const errorJson = JSON.parse(errorText); + const message = errorJson.message ?? "Unknown error"; + const code = errorJson.code; + + // Map known append errors (412 Precondition Failed) + if (status === 412) { + if ("seq_num_mismatch" in errorJson) { + const expected = Number(errorJson.seq_num_mismatch); + queueMicrotask(() => + safeError( + new SeqNumMismatchError({ + message: + "Append condition failed: sequence number mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedSeqNum: expected, + }), + ), + ); + } else if ( + "fencing_token_mismatch" in errorJson + ) { + const expected = String( + errorJson.fencing_token_mismatch, + ); + queueMicrotask(() => + safeError( + new FencingTokenMismatchError({ + message: + "Append condition failed: fencing token mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedFencingToken: expected, + }), + ), + ); + } else { + queueMicrotask(() => + safeError( + new S2Error({ message, code, status }), + ), + ); + } + } else { + queueMicrotask(() => + safeError(new S2Error({ message, code, status })), + ); + } + } catch { + const message = errorText || "Unknown error"; + queueMicrotask(() => + safeError(new S2Error({ message, status })), + ); + } + } + stream.close(); + } else { // Parse AppendAck try { const protoAck = ProtoAppendAck.fromBinary(frame.body); diff --git a/src/stream.ts b/src/stream.ts index 4f98b6d..0940622 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -68,7 +68,7 @@ export class S2Stream { ); }); - return response.data; + return response.data!; } /** diff --git a/src/tests/retryReadSession.test.ts b/src/tests/retryReadSession.test.ts index ead1859..043907d 100644 --- a/src/tests/retryReadSession.test.ts +++ b/src/tests/retryReadSession.test.ts @@ -413,7 +413,7 @@ describe("ReadSession (unit)", () => { expect(secondArgs.until).toBe(1000); // Unchanged (absolute boundary) }); - it("fails after max retry attempts exhausted", async () => { + it("fails after max retry attempts exhausted", async () => { let callCount = 0; const session = await ReadSession.create( @@ -440,6 +440,52 @@ describe("ReadSession (unit)", () => { }); // Should have tried 3 times (initial + 2 retries) - expect(callCount).toBe(3); - }); + expect(callCount).toBe(3); + }); + + it("does not double-subtract count across multiple retries", async () => { + // First attempt emits 30 then errors, second emits 40 then errors, third succeeds + const records1: ReadRecord<"string">[] = Array.from({ length: 30 }, (_, i) => ({ seq_num: i, timestamp: 0, body: "a" })); + const records2: ReadRecord<"string">[] = Array.from({ length: 40 }, (_, i) => ({ seq_num: 30 + i, timestamp: 0, body: "b" })); + + let call = 0; + const capturedArgs: Array> = []; + + const session = await ReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + call++; + if (call === 1) { + // First call: 30 records then error + return new FakeReadSession({ + records: records1, + errorAfterRecords: 30, + error: new S2Error({ message: "transient", status: 500 }), + }); + } else if (call === 2) { + // Second call: 40 records then error + return new FakeReadSession({ + records: records2, + errorAfterRecords: 40, + error: new S2Error({ message: "transient", status: 500 }), + }); + } + // Third call: success (no more records to emit; just close) + return new FakeReadSession({ records: [] }); + }, + { seq_num: 0, count: 100 }, + { retryBackoffDurationMs: 1, maxAttempts: 2 }, + ); + + // Drain the session + for await (const _ of session) { + // consuming until completion + } + + // Expect args progression: 100 -> 70 -> 30 + expect(capturedArgs).toHaveLength(3); + expect(capturedArgs[0]?.count).toBe(100); + expect(capturedArgs[1]?.count).toBe(70); // 100 - 30 + expect(capturedArgs[2]?.count).toBe(30); // 100 - (30 + 40) + }); }); From 7c0724df2aa2f75615e9854a0cac093ab5212374 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 15:35:13 -0800 Subject: [PATCH 16/26] s2Data --- examples/image.ts | 7 ++- examples/image2.ts | 144 -------------------------------------------- src/accessTokens.ts | 26 +++----- src/basins.ts | 40 +++++------- src/error.ts | 47 +++++++++++++++ src/lib/retry.ts | 53 +++++++++------- src/metrics.ts | 26 +++----- src/stream.ts | 10 ++- src/streams.ts | 40 +++++------- 9 files changed, 139 insertions(+), 254 deletions(-) delete mode 100644 examples/image2.ts diff --git a/examples/image.ts b/examples/image.ts index ea416d0..2d11912 100644 --- a/examples/image.ts +++ b/examples/image.ts @@ -31,6 +31,11 @@ function rechunkStream( const s2 = new S2({ accessToken: process.env.S2_ACCESS_TOKEN!, + retry: { + maxAttempts: 10, + retryBackoffDurationMs: 100, + appendRetryPolicy: "noSideEffects", + }, }); const basinName = process.env.S2_BASIN; @@ -45,7 +50,7 @@ const stream = basin.stream("image"); const startAt = await stream.checkTail(); const session = await stream.appendSession({ - maxQueuedBytes: 1024 * 1024 * 10, + maxQueuedBytes: 1024 * 1024, // 1MiB }); let image = await fetch( "https://upload.wikimedia.org/wikipedia/commons/2/24/Peter_Paul_Rubens_-_Self-portrait_-_RH.S.180_-_Rubenshuis_%28after_restoration%29.jpg", diff --git a/examples/image2.ts b/examples/image2.ts deleted file mode 100644 index 39ecb1e..0000000 --- a/examples/image2.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { createWriteStream } from "node:fs"; -import { - AppendRecord, - BatchTransform, - type ReadRecord, - S2, -} from "../src/index.js"; - -function rechunkStream( - desiredChunkSize: number, -): TransformStream { - let buffer = new Uint8Array(0); - return new TransformStream({ - transform(chunk, controller) { - const newBuffer = new Uint8Array(buffer.length + chunk.length); - newBuffer.set(buffer); - newBuffer.set(chunk, buffer.length); - buffer = newBuffer; - while (buffer.length >= desiredChunkSize) { - controller.enqueue(buffer.slice(0, desiredChunkSize)); - buffer = buffer.slice(desiredChunkSize); - } - }, - flush(controller) { - if (buffer.length > 0) { - controller.enqueue(buffer); - } - }, - }); -} - -// const s2 = new S2({ -// accessToken: process.env.S2_ACCESS_TOKEN!, -// }); - -const s2 = new S2({ - accessToken: process.env.S2_ACCESS_TOKEN!, - baseUrl: `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, - makeBasinBaseUrl: (basinName) => - `https://${process.env.S2_ACCOUNT_ENDPOINT!}/v1`, - retry: { - maxAttempts: 10, - retryBackoffDurationMs: 100, - appendRetryPolicy: "all", - requestTimeoutMillis: 20000, - }, -}); - -const basinName = process.env.S2_BASIN; -if (!basinName) { - console.error("S2_BASIN environment variable is not set"); - process.exit(1); -} - -const basin = s2.basin(process.env.S2_BASIN!); -const stream = basin.stream("image"); - -console.log("ct"); -const startAt = await stream.checkTail(); - -console.log(`Tail seqNum=${startAt.tail.seq_num}`); - - -const session = await stream.appendSession({ - maxQueuedBytes: 1024 * 1024 * 1, - maxInflightBatches: 10, -}); -let image = await fetch( - "https://upload.wikimedia.org/wikipedia/commons/2/24/Peter_Paul_Rubens_-_Self-portrait_-_RH.S.180_-_Rubenshuis_%28after_restoration%29.jpg", -); - -// Write directly from fetch response to S2 stream -let append = await image - .body! // Ensure each chunk is at most 128KiB. S2 has a maximum individual record size of 1MiB. - .pipeThrough(rechunkStream(1024 * 128)) - // Convert each chunk to an AppendRecord. - .pipeThrough( - new TransformStream({ - transform(arr, controller) { - controller.enqueue(AppendRecord.make(arr)); - }, - }), - ) - // Collect records into batches. - .pipeThrough( - new BatchTransform({ - lingerDurationMillis: 50, - match_seq_num: startAt.tail.seq_num, - }), - ) - // Write to the S2 stream. - .pipeTo(session.writable); - -console.log( - `image written to S2 over ${session.lastAckedPosition()!.end!.seq_num - startAt.tail.seq_num} records, starting at seqNum=${startAt.tail.seq_num}`, -); - -let lastAcked= session.lastAckedPosition()!.end.seq_num; -console.log("lastAcked=%s", { lastAcked }) - -let readSession = await stream.readSession({ - seq_num: startAt.tail.seq_num, - count: lastAcked - startAt.tail.seq_num, - as: "bytes", -}); - -// Write to a local file. -const id = Math.random().toString(36).slice(2, 10); -// Use a larger buffer (default is 16KB, we use 512KB) -const out = createWriteStream(`image-${id}.jpg`, { - highWaterMark: 512 * 1024, // 512KB buffer -}); - -await readSession - .pipeThrough( - new TransformStream, Uint8Array>({ - transform(arr, controller) { - controller.enqueue(arr.body); - }, - }), - ) - .pipeTo( - new WritableStream({ - async write(chunk) { - // Handle backpressure - wait if buffer is full - if (!out.write(chunk)) { - await new Promise((resolve) => out.once("drain", resolve)); - } - }, - // Don't close here - we'll close manually after to ensure flush - }), - ); - -// Ensure the file is fully written and closed before exiting -await new Promise((resolve, reject) => { - out.close((err) => { - if (err) reject(err); - else resolve(); - }); -}); - -console.log(`Image written to image-${id}.jpg`); - -process.exit(0); diff --git a/src/accessTokens.ts b/src/accessTokens.ts index a5f2556..c1a2df6 100644 --- a/src/accessTokens.ts +++ b/src/accessTokens.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Error } from "./error.js"; +import { S2Error, withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type IssueAccessTokenData, @@ -35,17 +35,15 @@ export class S2AccessTokens { * @param args.limit Max results (up to 1000) */ public async list(args?: ListAccessTokensArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => listAccessTokens({ client: this.client, query: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -57,17 +55,15 @@ export class S2AccessTokens { * @param args.expires_at Expiration in ISO 8601; defaults to requestor's token expiry */ public async issue(args: IssueAccessTokenArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => issueAccessToken({ client: this.client, body: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -76,16 +72,14 @@ export class S2AccessTokens { * @param args.id Token ID to revoke */ public async revoke(args: RevokeAccessTokenArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => revokeAccessToken({ client: this.client, path: args, ...options, }), ); - }); - - return response.data; + }); } } diff --git a/src/basins.ts b/src/basins.ts index 10f80c6..30b4610 100644 --- a/src/basins.ts +++ b/src/basins.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Error } from "./error.js"; +import { S2Error, withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type BasinConfig, @@ -46,17 +46,15 @@ export class S2Basins { args?: ListBasinsArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => listBasins({ client: this.client, query: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -70,17 +68,15 @@ export class S2Basins { args: CreateBasinArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => createBasin({ client: this.client, body: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -92,17 +88,15 @@ export class S2Basins { args: GetBasinConfigArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => getBasinConfig({ client: this.client, path: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -114,15 +108,15 @@ export class S2Basins { args: DeleteBasinArgs, options?: S2RequestOptions, ): Promise { - await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + await withRetries(this.retryConfig, async () => { + return await withS2Data(() => deleteBasin({ client: this.client, path: args, ...options, }), ); - }); + }); } /** @@ -135,8 +129,8 @@ export class S2Basins { args: ReconfigureBasinArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => reconfigureBasin({ client: this.client, path: args, @@ -144,8 +138,6 @@ export class S2Basins { ...options, }), ); - }); - - return response.data; + }); } } diff --git a/src/error.ts b/src/error.ts index bd8de15..837b9af 100644 --- a/src/error.ts +++ b/src/error.ts @@ -105,6 +105,53 @@ export async function withS2Error(fn: () => Promise): Promise { } } +/** + * Execute a generated client call and return its `data` on success. + * Throws S2Error when the response contains `error`, or when the + * response has no `data` and is not a 204 No Content. + */ +export async function withS2Data( + fn: () => Promise<{ + data?: T; + error?: unknown; + response?: { status?: number; statusText?: string }; + } | T>, +): Promise { + try { + const res: any = await fn(); + if ( + res && + typeof res === "object" && + (Object.prototype.hasOwnProperty.call(res, "error") || + Object.prototype.hasOwnProperty.call(res, "data") || + Object.prototype.hasOwnProperty.call(res, "response")) + ) { + const status = res.response?.status as number | undefined; + const statusText = res.response?.statusText as string | undefined; + if (res.error) { + const err = res.error; + if (typeof err === "object" && "message" in err) { + throw new S2Error({ + message: (err as any).message ?? statusText ?? "Error", + code: (err as any).code ?? undefined, + status, + }); + } + throw new S2Error({ message: statusText ?? "Request failed", status }); + } + // No error + if (typeof res.data !== "undefined") return res.data as T; + // Treat 204 as success for void endpoints + if (status === 204) return undefined as T; + throw new S2Error({ message: "Empty response", status }); + } + // Not a generated client response; return as-is + return res as T; + } catch (error) { + throw s2Error(error); + } +} + /** * Rich error type used by the SDK to surface HTTP and protocol errors. * diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 39a2036..55b9120 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -210,29 +210,36 @@ export class ReadSession< // Check if we can retry (track session attempts, not record reads) if (isRetryable(error) && attempt < retryConfig.maxAttempts) { - if (this._nextReadPosition) { - nextArgs.seq_num = this._nextReadPosition.seq_num; - } - // Recompute remaining budget from original request each time to avoid double-subtraction - if (baselineCount !== undefined) { - const remaining = Math.max(0, baselineCount - this._recordsRead); - nextArgs.count = remaining as any; - } - if (baselineBytes !== undefined) { - const remaining = Math.max(0, baselineBytes - this._bytesRead); - nextArgs.bytes = remaining as any; - } - // Adjust wait from original budget based on total elapsed time since start - if (baselineWait !== undefined) { - const elapsedSeconds = (performance.now() - startTimeMs) / 1000; - nextArgs.wait = Math.max(0, baselineWait - elapsedSeconds) as any; - } - const delay = calculateDelay( - attempt, - retryConfig.retryBackoffDurationMs, - ); - debug("will retry after %dms, status=%s", delay, error.status); - await sleep(delay); + if (this._nextReadPosition) { + nextArgs.seq_num = this._nextReadPosition.seq_num as any; + // Clear alternative start position fields to avoid conflicting params + delete (nextArgs as any).timestamp; + delete (nextArgs as any).tail_offset; + } + // Compute planned backoff delay now so we can subtract it from wait budget + const delay = calculateDelay( + attempt, + retryConfig.retryBackoffDurationMs, + ); + // Recompute remaining budget from original request each time to avoid double-subtraction + if (baselineCount !== undefined) { + const remaining = Math.max(0, baselineCount - this._recordsRead); + nextArgs.count = remaining as any; + } + if (baselineBytes !== undefined) { + const remaining = Math.max(0, baselineBytes - this._bytesRead); + nextArgs.bytes = remaining as any; + } + // Adjust wait from original budget based on total elapsed time since start + if (baselineWait !== undefined) { + const elapsedSeconds = (performance.now() - startTimeMs) / 1000; + nextArgs.wait = Math.max( + 0, + baselineWait - (elapsedSeconds + delay / 1000), + ) as any; + } + debug("will retry after %dms, status=%s", delay, error.status); + await sleep(delay); attempt++; break; // Break inner loop to retry } diff --git a/src/metrics.ts b/src/metrics.ts index 5eb0123..54382f5 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Error } from "./error.js"; +import { S2Error, withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AccountMetricsData, @@ -33,17 +33,15 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async account(args: AccountMetricsArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => accountMetrics({ client: this.client, query: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -56,8 +54,8 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async basin(args: BasinMetricsArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => basinMetrics({ client: this.client, path: args, @@ -65,9 +63,7 @@ export class S2Metrics { ...options, }), ); - }); - - return response.data; + }); } /** @@ -81,8 +77,8 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async stream(args: StreamMetricsArgs, options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => streamMetrics({ client: this.client, path: args, @@ -90,8 +86,6 @@ export class S2Metrics { ...options, }), ); - }); - - return response.data; + }); } } diff --git a/src/stream.ts b/src/stream.ts index 0940622..ee8d014 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,5 +1,5 @@ import type { RetryConfig, S2RequestOptions } from "./common.js"; -import { withS2Error } from "./error.js"; +import { withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AppendAck, checkTail } from "./generated/index.js"; import { isRetryable, withRetries } from "./lib/retry.js"; @@ -56,8 +56,8 @@ export class S2Stream { * Returns the next sequence number and timestamp to be assigned (`tail`). */ public async checkTail(options?: S2RequestOptions) { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => checkTail({ client: this.client, path: { @@ -66,9 +66,7 @@ export class S2Stream { ...options, }), ); - }); - - return response.data!; + }); } /** diff --git a/src/streams.ts b/src/streams.ts index 65157d8..b5f6586 100644 --- a/src/streams.ts +++ b/src/streams.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Error } from "./error.js"; +import { S2Error, withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type CreateStreamData, @@ -47,17 +47,15 @@ export class S2Streams { args?: ListStreamsArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => listStreams({ client: this.client, query: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -70,17 +68,15 @@ export class S2Streams { args: CreateStreamArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => createStream({ client: this.client, body: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -92,17 +88,15 @@ export class S2Streams { args: GetStreamConfigArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => getStreamConfig({ client: this.client, path: args, ...options, }), ); - }); - - return response.data; + }); } /** @@ -114,15 +108,15 @@ export class S2Streams { args: DeleteStreamArgs, options?: S2RequestOptions, ): Promise { - await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + await withRetries(this.retryConfig, async () => { + return await withS2Data(() => deleteStream({ client: this.client, path: args, ...options, }), ); - }); + }); } /** @@ -135,8 +129,8 @@ export class S2Streams { args: ReconfigureStreamArgs, options?: S2RequestOptions, ): Promise { - const response = await withRetries(this.retryConfig, async () => { - return await withS2Error(async () => + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => reconfigureStream({ client: this.client, path: args, @@ -144,8 +138,6 @@ export class S2Streams { ...options, }), ); - }); - - return response.data; + }); } } From 744f1f224dc0552deb2c4e919a2663e04ac16f2e Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 16:20:54 -0800 Subject: [PATCH 17/26] idempotency --- src/accessTokens.ts | 54 +++---- src/basins.ts | 92 ++++++------ src/error.ts | 147 +++++++++---------- src/lib/retry.ts | 75 +++++----- src/lib/stream/transport/s2s/index.ts | 200 +++++++++++++------------- src/metrics.ts | 58 ++++---- src/s2.ts | 12 +- src/stream.ts | 29 ++-- src/streams.ts | 92 ++++++------ src/tests/retryReadSession.test.ts | 104 +++++++------- src/tests/withS2Error.test.ts | 107 +++++++------- 11 files changed, 487 insertions(+), 483 deletions(-) diff --git a/src/accessTokens.ts b/src/accessTokens.ts index c1a2df6..bdb6eb1 100644 --- a/src/accessTokens.ts +++ b/src/accessTokens.ts @@ -35,15 +35,15 @@ export class S2AccessTokens { * @param args.limit Max results (up to 1000) */ public async list(args?: ListAccessTokensArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - listAccessTokens({ - client: this.client, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + listAccessTokens({ + client: this.client, + query: args, + ...options, + }), + ); + }); } /** @@ -55,15 +55,15 @@ export class S2AccessTokens { * @param args.expires_at Expiration in ISO 8601; defaults to requestor's token expiry */ public async issue(args: IssueAccessTokenArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - issueAccessToken({ - client: this.client, - body: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + issueAccessToken({ + client: this.client, + body: args, + ...options, + }), + ); + }); } /** @@ -72,14 +72,14 @@ export class S2AccessTokens { * @param args.id Token ID to revoke */ public async revoke(args: RevokeAccessTokenArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - revokeAccessToken({ - client: this.client, - path: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + revokeAccessToken({ + client: this.client, + path: args, + ...options, + }), + ); + }); } } diff --git a/src/basins.ts b/src/basins.ts index 30b4610..44ad08f 100644 --- a/src/basins.ts +++ b/src/basins.ts @@ -46,15 +46,15 @@ export class S2Basins { args?: ListBasinsArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - listBasins({ - client: this.client, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + listBasins({ + client: this.client, + query: args, + ...options, + }), + ); + }); } /** @@ -68,15 +68,15 @@ export class S2Basins { args: CreateBasinArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - createBasin({ - client: this.client, - body: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + createBasin({ + client: this.client, + body: args, + ...options, + }), + ); + }); } /** @@ -88,15 +88,15 @@ export class S2Basins { args: GetBasinConfigArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - getBasinConfig({ - client: this.client, - path: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + getBasinConfig({ + client: this.client, + path: args, + ...options, + }), + ); + }); } /** @@ -108,15 +108,15 @@ export class S2Basins { args: DeleteBasinArgs, options?: S2RequestOptions, ): Promise { - await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - deleteBasin({ - client: this.client, - path: args, - ...options, - }), - ); - }); + await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + deleteBasin({ + client: this.client, + path: args, + ...options, + }), + ); + }); } /** @@ -129,15 +129,15 @@ export class S2Basins { args: ReconfigureBasinArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - reconfigureBasin({ - client: this.client, - path: args, - body: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + reconfigureBasin({ + client: this.client, + path: args, + body: args, + ...options, + }), + ); + }); } } diff --git a/src/error.ts b/src/error.ts index 837b9af..2c14396 100644 --- a/src/error.ts +++ b/src/error.ts @@ -64,45 +64,43 @@ export function s2Error(error: any): S2Error { } export async function withS2Error(fn: () => Promise): Promise { - try { - const result: any = await fn(); + try { + const result: any = await fn(); - // Support response-parsing mode (throwOnError=false): - // Generated client responses have shape { data, error?, response } - if ( - result && - typeof result === "object" && - Object.prototype.hasOwnProperty.call(result, "error") - ) { - const err = result.error; - if (err) { - const status = result.response?.status as number | undefined; - const statusText = result.response?.statusText as - | string - | undefined; + // Support response-parsing mode (throwOnError=false): + // Generated client responses have shape { data, error?, response } + if ( + result && + typeof result === "object" && + Object.prototype.hasOwnProperty.call(result, "error") + ) { + const err = result.error; + if (err) { + const status = result.response?.status as number | undefined; + const statusText = result.response?.statusText as string | undefined; - // If server provided structured error with message/code, use it - if (typeof err === "object" && "message" in err) { - throw new S2Error({ - message: (err as any).message ?? statusText ?? "Error", - code: (err as any).code ?? undefined, - status, - }); - } + // If server provided structured error with message/code, use it + if (typeof err === "object" && "message" in err) { + throw new S2Error({ + message: (err as any).message ?? statusText ?? "Error", + code: (err as any).code ?? undefined, + status, + }); + } - // Fallback: synthesize from HTTP response metadata - throw new S2Error({ - message: statusText ?? "Request failed", - status, - }); - } - } + // Fallback: synthesize from HTTP response metadata + throw new S2Error({ + message: statusText ?? "Request failed", + status, + }); + } + } - return result as T; - } catch (error) { - // Network and other thrown errors - throw s2Error(error); - } + return result as T; + } catch (error) { + // Network and other thrown errors + throw s2Error(error); + } } /** @@ -111,45 +109,48 @@ export async function withS2Error(fn: () => Promise): Promise { * response has no `data` and is not a 204 No Content. */ export async function withS2Data( - fn: () => Promise<{ - data?: T; - error?: unknown; - response?: { status?: number; statusText?: string }; - } | T>, + fn: () => Promise< + | { + data?: T; + error?: unknown; + response?: { status?: number; statusText?: string }; + } + | T + >, ): Promise { - try { - const res: any = await fn(); - if ( - res && - typeof res === "object" && - (Object.prototype.hasOwnProperty.call(res, "error") || - Object.prototype.hasOwnProperty.call(res, "data") || - Object.prototype.hasOwnProperty.call(res, "response")) - ) { - const status = res.response?.status as number | undefined; - const statusText = res.response?.statusText as string | undefined; - if (res.error) { - const err = res.error; - if (typeof err === "object" && "message" in err) { - throw new S2Error({ - message: (err as any).message ?? statusText ?? "Error", - code: (err as any).code ?? undefined, - status, - }); - } - throw new S2Error({ message: statusText ?? "Request failed", status }); - } - // No error - if (typeof res.data !== "undefined") return res.data as T; - // Treat 204 as success for void endpoints - if (status === 204) return undefined as T; - throw new S2Error({ message: "Empty response", status }); - } - // Not a generated client response; return as-is - return res as T; - } catch (error) { - throw s2Error(error); - } + try { + const res: any = await fn(); + if ( + res && + typeof res === "object" && + (Object.prototype.hasOwnProperty.call(res, "error") || + Object.prototype.hasOwnProperty.call(res, "data") || + Object.prototype.hasOwnProperty.call(res, "response")) + ) { + const status = res.response?.status as number | undefined; + const statusText = res.response?.statusText as string | undefined; + if (res.error) { + const err = res.error; + if (typeof err === "object" && "message" in err) { + throw new S2Error({ + message: (err as any).message ?? statusText ?? "Error", + code: (err as any).code ?? undefined, + status, + }); + } + throw new S2Error({ message: statusText ?? "Request failed", status }); + } + // No error + if (typeof res.data !== "undefined") return res.data as T; + // Treat 204 as success for void endpoints + if (status === 204) return undefined as T; + throw new S2Error({ message: "Empty response", status }); + } + // Not a generated client response; return as-is + return res as T; + } catch (error) { + throw s2Error(error); + } } /** diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 55b9120..8c4547d 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -210,36 +210,43 @@ export class ReadSession< // Check if we can retry (track session attempts, not record reads) if (isRetryable(error) && attempt < retryConfig.maxAttempts) { - if (this._nextReadPosition) { - nextArgs.seq_num = this._nextReadPosition.seq_num as any; - // Clear alternative start position fields to avoid conflicting params - delete (nextArgs as any).timestamp; - delete (nextArgs as any).tail_offset; - } - // Compute planned backoff delay now so we can subtract it from wait budget - const delay = calculateDelay( - attempt, - retryConfig.retryBackoffDurationMs, - ); - // Recompute remaining budget from original request each time to avoid double-subtraction - if (baselineCount !== undefined) { - const remaining = Math.max(0, baselineCount - this._recordsRead); - nextArgs.count = remaining as any; - } - if (baselineBytes !== undefined) { - const remaining = Math.max(0, baselineBytes - this._bytesRead); - nextArgs.bytes = remaining as any; - } - // Adjust wait from original budget based on total elapsed time since start - if (baselineWait !== undefined) { - const elapsedSeconds = (performance.now() - startTimeMs) / 1000; - nextArgs.wait = Math.max( - 0, - baselineWait - (elapsedSeconds + delay / 1000), - ) as any; - } - debug("will retry after %dms, status=%s", delay, error.status); - await sleep(delay); + if (this._nextReadPosition) { + nextArgs.seq_num = this._nextReadPosition.seq_num as any; + // Clear alternative start position fields to avoid conflicting params + delete (nextArgs as any).timestamp; + delete (nextArgs as any).tail_offset; + } + // Compute planned backoff delay now so we can subtract it from wait budget + const delay = calculateDelay( + attempt, + retryConfig.retryBackoffDurationMs, + ); + // Recompute remaining budget from original request each time to avoid double-subtraction + if (baselineCount !== undefined) { + const remaining = Math.max( + 0, + baselineCount - this._recordsRead, + ); + nextArgs.count = remaining as any; + } + if (baselineBytes !== undefined) { + const remaining = Math.max( + 0, + baselineBytes - this._bytesRead, + ); + nextArgs.bytes = remaining as any; + } + // Adjust wait from original budget based on total elapsed time since start + if (baselineWait !== undefined) { + const elapsedSeconds = + (performance.now() - startTimeMs) / 1000; + nextArgs.wait = Math.max( + 0, + baselineWait - (elapsedSeconds + delay / 1000), + ) as any; + } + debug("will retry after %dms, status=%s", delay, error.status); + await sleep(delay); attempt++; break; // Break inner loop to retry } @@ -851,7 +858,7 @@ export class AppendSession implements AsyncDisposable { // Check policy compliance if ( this.retryConfig.appendRetryPolicy === "noSideEffects" && - !this.isAppendRetryAllowed(head) + !this.isIdempotent(head) ) { debug("error not policy-compliant (noSideEffects), aborting"); await this.abort(error); @@ -971,13 +978,13 @@ export class AppendSession implements AsyncDisposable { /** * Check if append can be retried under noSideEffects policy. + * For appends, idempotency requires match_seq_num. */ - private isAppendRetryAllowed(entry: InflightEntry): boolean { + private isIdempotent(entry: InflightEntry): boolean { const args = entry.args; if (!args) return false; - // Allow retry if match_seq_num or fencing_token is set (idempotent) - return args.match_seq_num !== undefined || args.fencing_token !== undefined; + return args.match_seq_num !== undefined; } /** diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index b8ee4ef..7481b74 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -8,6 +8,12 @@ import * as http2 from "node:http2"; import createDebug from "debug"; import type { S2RequestOptions } from "../../../../common.js"; +import { + FencingTokenMismatchError, + RangeNotSatisfiableError, + S2Error, + SeqNumMismatchError, +} from "../../../../error.js"; import type { AppendAck, StreamPosition } from "../../../../generated/index.js"; import { AppendAck as ProtoAppendAck, @@ -15,12 +21,6 @@ import { ReadBatch as ProtoReadBatch, type StreamPosition as ProtoStreamPosition, } from "../../../../generated/proto/s2.js"; -import { - FencingTokenMismatchError, - RangeNotSatisfiableError, - S2Error, - SeqNumMismatchError, -} from "../../../../error.js"; import { meteredSizeBytes } from "../../../../utils.js"; import * as Redacted from "../../../redacted.js"; import type { AppendResult, CloseResult } from "../../../result.js"; @@ -389,42 +389,40 @@ class S2SReadSession let frame = parser.parseFrame(); while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); - try { - const errorJson = JSON.parse(errorText); - const status = frame.statusCode ?? 500; - const message = errorJson.message ?? "Unknown error"; - const code = errorJson.code; - - // Map known read errors - if (status === 416) { - safeError( - new RangeNotSatisfiableError({ status }), - ); - } else { - safeError( - new S2Error({ - message, - code, - status, - }), - ); - } - } catch { - safeError( - new S2Error({ - message: errorText || "Unknown error", - status: frame.statusCode, - }), - ); - } - } else { - safeClose(); - } - stream.close(); - } else { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + try { + const errorJson = JSON.parse(errorText); + const status = frame.statusCode ?? 500; + const message = errorJson.message ?? "Unknown error"; + const code = errorJson.code; + + // Map known read errors + if (status === 416) { + safeError(new RangeNotSatisfiableError({ status })); + } else { + safeError( + new S2Error({ + message, + code, + status, + }), + ); + } + } catch { + safeError( + new S2Error({ + message: errorText || "Unknown error", + status: frame.statusCode, + }), + ); + } + } else { + safeClose(); + } + stream.close(); + } else { // Parse ReadBatch try { const protoBatch = ProtoReadBatch.fromBinary(frame.body); @@ -708,68 +706,62 @@ class S2SAppendSession { let frame = this.parser.parseFrame(); while (frame) { - if (frame.terminal) { - if (frame.statusCode && frame.statusCode >= 400) { - const errorText = textDecoder.decode(frame.body); - const status = frame.statusCode ?? 500; - try { - const errorJson = JSON.parse(errorText); - const message = errorJson.message ?? "Unknown error"; - const code = errorJson.code; - - // Map known append errors (412 Precondition Failed) - if (status === 412) { - if ("seq_num_mismatch" in errorJson) { - const expected = Number(errorJson.seq_num_mismatch); - queueMicrotask(() => - safeError( - new SeqNumMismatchError({ - message: - "Append condition failed: sequence number mismatch", - code: "APPEND_CONDITION_FAILED", - status, - expectedSeqNum: expected, - }), - ), - ); - } else if ( - "fencing_token_mismatch" in errorJson - ) { - const expected = String( - errorJson.fencing_token_mismatch, - ); - queueMicrotask(() => - safeError( - new FencingTokenMismatchError({ - message: - "Append condition failed: fencing token mismatch", - code: "APPEND_CONDITION_FAILED", - status, - expectedFencingToken: expected, - }), - ), - ); - } else { - queueMicrotask(() => - safeError( - new S2Error({ message, code, status }), - ), - ); - } - } else { - queueMicrotask(() => - safeError(new S2Error({ message, code, status })), - ); - } - } catch { - const message = errorText || "Unknown error"; - queueMicrotask(() => - safeError(new S2Error({ message, status })), - ); - } - } - stream.close(); - } else { + if (frame.terminal) { + if (frame.statusCode && frame.statusCode >= 400) { + const errorText = textDecoder.decode(frame.body); + const status = frame.statusCode ?? 500; + try { + const errorJson = JSON.parse(errorText); + const message = errorJson.message ?? "Unknown error"; + const code = errorJson.code; + + // Map known append errors (412 Precondition Failed) + if (status === 412) { + if ("seq_num_mismatch" in errorJson) { + const expected = Number(errorJson.seq_num_mismatch); + queueMicrotask(() => + safeError( + new SeqNumMismatchError({ + message: + "Append condition failed: sequence number mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedSeqNum: expected, + }), + ), + ); + } else if ("fencing_token_mismatch" in errorJson) { + const expected = String(errorJson.fencing_token_mismatch); + queueMicrotask(() => + safeError( + new FencingTokenMismatchError({ + message: + "Append condition failed: fencing token mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedFencingToken: expected, + }), + ), + ); + } else { + queueMicrotask(() => + safeError(new S2Error({ message, code, status })), + ); + } + } else { + queueMicrotask(() => + safeError(new S2Error({ message, code, status })), + ); + } + } catch { + const message = errorText || "Unknown error"; + queueMicrotask(() => + safeError(new S2Error({ message, status })), + ); + } + } + stream.close(); + } else { // Parse AppendAck try { const protoAck = ProtoAppendAck.fromBinary(frame.body); diff --git a/src/metrics.ts b/src/metrics.ts index 54382f5..c25f063 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -33,15 +33,15 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async account(args: AccountMetricsArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - accountMetrics({ - client: this.client, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + accountMetrics({ + client: this.client, + query: args, + ...options, + }), + ); + }); } /** @@ -54,16 +54,16 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async basin(args: BasinMetricsArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - basinMetrics({ - client: this.client, - path: args, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + basinMetrics({ + client: this.client, + path: args, + query: args, + ...options, + }), + ); + }); } /** @@ -77,15 +77,15 @@ export class S2Metrics { * @param args.interval Optional aggregation interval for timeseries sets */ public async stream(args: StreamMetricsArgs, options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - streamMetrics({ - client: this.client, - path: args, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + streamMetrics({ + client: this.client, + path: args, + query: args, + ...options, + }), + ); + }); } } diff --git a/src/s2.ts b/src/s2.ts index 62e3341..1c1acb6 100644 --- a/src/s2.ts +++ b/src/s2.ts @@ -43,12 +43,12 @@ export class S2 { constructor(options: S2ClientOptions) { this.accessToken = Redacted.make(options.accessToken); this.retryConfig = options.retry ?? {}; - this.client = createClient( - createConfig({ - baseUrl: options.baseUrl ?? defaultBaseUrl, - auth: () => Redacted.value(this.accessToken), - }), - ); + this.client = createClient( + createConfig({ + baseUrl: options.baseUrl ?? defaultBaseUrl, + auth: () => Redacted.value(this.accessToken), + }), + ); this.client.interceptors.error.use((err, res, req, opt) => { return new S2Error({ diff --git a/src/stream.ts b/src/stream.ts index ee8d014..abead14 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -56,17 +56,17 @@ export class S2Stream { * Returns the next sequence number and timestamp to be assigned (`tail`). */ public async checkTail(options?: S2RequestOptions) { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - checkTail({ - client: this.client, - path: { - stream: this.name, - }, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + checkTail({ + client: this.client, + path: { + stream: this.name, + }, + ...options, + }), + ); + }); } /** @@ -117,10 +117,9 @@ export class S2Stream { }, (config, error) => { if ((config.appendRetryPolicy ?? "noSideEffects") === "noSideEffects") { - // Allow retry if the append is idempotent (match_seq_num or fencing_token) - // or if the error qualifies as retryable by shared logic. - const isIdempotent = !!args?.match_seq_num || !!args?.fencing_token; - return isIdempotent || isRetryable(error); + // Allow retry only when the append is naturally idempotent by containing + // a match_seq_num condition. + return !!args?.match_seq_num; } else { return true; } diff --git a/src/streams.ts b/src/streams.ts index b5f6586..637499a 100644 --- a/src/streams.ts +++ b/src/streams.ts @@ -47,15 +47,15 @@ export class S2Streams { args?: ListStreamsArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - listStreams({ - client: this.client, - query: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + listStreams({ + client: this.client, + query: args, + ...options, + }), + ); + }); } /** @@ -68,15 +68,15 @@ export class S2Streams { args: CreateStreamArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - createStream({ - client: this.client, - body: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + createStream({ + client: this.client, + body: args, + ...options, + }), + ); + }); } /** @@ -88,15 +88,15 @@ export class S2Streams { args: GetStreamConfigArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - getStreamConfig({ - client: this.client, - path: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + getStreamConfig({ + client: this.client, + path: args, + ...options, + }), + ); + }); } /** @@ -108,15 +108,15 @@ export class S2Streams { args: DeleteStreamArgs, options?: S2RequestOptions, ): Promise { - await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - deleteStream({ - client: this.client, - path: args, - ...options, - }), - ); - }); + await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + deleteStream({ + client: this.client, + path: args, + ...options, + }), + ); + }); } /** @@ -129,15 +129,15 @@ export class S2Streams { args: ReconfigureStreamArgs, options?: S2RequestOptions, ): Promise { - return await withRetries(this.retryConfig, async () => { - return await withS2Data(() => - reconfigureStream({ - client: this.client, - path: args, - body: args, - ...options, - }), - ); - }); + return await withRetries(this.retryConfig, async () => { + return await withS2Data(() => + reconfigureStream({ + client: this.client, + path: args, + body: args, + ...options, + }), + ); + }); } } diff --git a/src/tests/retryReadSession.test.ts b/src/tests/retryReadSession.test.ts index 043907d..2ab7ec7 100644 --- a/src/tests/retryReadSession.test.ts +++ b/src/tests/retryReadSession.test.ts @@ -413,7 +413,7 @@ describe("ReadSession (unit)", () => { expect(secondArgs.until).toBe(1000); // Unchanged (absolute boundary) }); - it("fails after max retry attempts exhausted", async () => { + it("fails after max retry attempts exhausted", async () => { let callCount = 0; const session = await ReadSession.create( @@ -440,52 +440,58 @@ describe("ReadSession (unit)", () => { }); // Should have tried 3 times (initial + 2 retries) - expect(callCount).toBe(3); - }); - - it("does not double-subtract count across multiple retries", async () => { - // First attempt emits 30 then errors, second emits 40 then errors, third succeeds - const records1: ReadRecord<"string">[] = Array.from({ length: 30 }, (_, i) => ({ seq_num: i, timestamp: 0, body: "a" })); - const records2: ReadRecord<"string">[] = Array.from({ length: 40 }, (_, i) => ({ seq_num: 30 + i, timestamp: 0, body: "b" })); - - let call = 0; - const capturedArgs: Array> = []; - - const session = await ReadSession.create( - async (args) => { - capturedArgs.push({ ...args }); - call++; - if (call === 1) { - // First call: 30 records then error - return new FakeReadSession({ - records: records1, - errorAfterRecords: 30, - error: new S2Error({ message: "transient", status: 500 }), - }); - } else if (call === 2) { - // Second call: 40 records then error - return new FakeReadSession({ - records: records2, - errorAfterRecords: 40, - error: new S2Error({ message: "transient", status: 500 }), - }); - } - // Third call: success (no more records to emit; just close) - return new FakeReadSession({ records: [] }); - }, - { seq_num: 0, count: 100 }, - { retryBackoffDurationMs: 1, maxAttempts: 2 }, - ); - - // Drain the session - for await (const _ of session) { - // consuming until completion - } - - // Expect args progression: 100 -> 70 -> 30 - expect(capturedArgs).toHaveLength(3); - expect(capturedArgs[0]?.count).toBe(100); - expect(capturedArgs[1]?.count).toBe(70); // 100 - 30 - expect(capturedArgs[2]?.count).toBe(30); // 100 - (30 + 40) - }); + expect(callCount).toBe(3); + }); + + it("does not double-subtract count across multiple retries", async () => { + // First attempt emits 30 then errors, second emits 40 then errors, third succeeds + const records1: ReadRecord<"string">[] = Array.from( + { length: 30 }, + (_, i) => ({ seq_num: i, timestamp: 0, body: "a" }), + ); + const records2: ReadRecord<"string">[] = Array.from( + { length: 40 }, + (_, i) => ({ seq_num: 30 + i, timestamp: 0, body: "b" }), + ); + + let call = 0; + const capturedArgs: Array> = []; + + const session = await ReadSession.create( + async (args) => { + capturedArgs.push({ ...args }); + call++; + if (call === 1) { + // First call: 30 records then error + return new FakeReadSession({ + records: records1, + errorAfterRecords: 30, + error: new S2Error({ message: "transient", status: 500 }), + }); + } else if (call === 2) { + // Second call: 40 records then error + return new FakeReadSession({ + records: records2, + errorAfterRecords: 40, + error: new S2Error({ message: "transient", status: 500 }), + }); + } + // Third call: success (no more records to emit; just close) + return new FakeReadSession({ records: [] }); + }, + { seq_num: 0, count: 100 }, + { retryBackoffDurationMs: 1, maxAttempts: 2 }, + ); + + // Drain the session + for await (const _ of session) { + // consuming until completion + } + + // Expect args progression: 100 -> 70 -> 30 + expect(capturedArgs).toHaveLength(3); + expect(capturedArgs[0]?.count).toBe(100); + expect(capturedArgs[1]?.count).toBe(70); // 100 - 30 + expect(capturedArgs[2]?.count).toBe(30); // 100 - (30 + 40) + }); }); diff --git a/src/tests/withS2Error.test.ts b/src/tests/withS2Error.test.ts index 1310bc1..0b59cce 100644 --- a/src/tests/withS2Error.test.ts +++ b/src/tests/withS2Error.test.ts @@ -2,58 +2,57 @@ import { describe, expect, it } from "vitest"; import { S2Error, withS2Error } from "../error.js"; describe("withS2Error response parsing", () => { - it("returns result when response has no error", async () => { - const value = await withS2Error(async () => ({ - data: { ok: 1 }, - error: undefined, - response: { status: 200, statusText: "OK" }, - })); - - expect(value).toMatchObject({ data: { ok: 1 }, response: { status: 200 } }); - }); - - it("throws S2Error with message/code/status when response.error has message", async () => { - const run = () => - withS2Error(async () => ({ - data: undefined, - error: { message: "Bad things", code: "BAD_THING" }, - response: { status: 400, statusText: "Bad Request" }, - })); - - await expect(run()).rejects.toMatchObject({ - name: "S2Error", - message: "Bad things", - code: "BAD_THING", - status: 400, - }); - }); - - it("falls back to HTTP statusText when error lacks message", async () => { - const run = () => - withS2Error(async () => ({ - data: undefined, - error: { something: "else" }, - response: { status: 502, statusText: "Bad Gateway" }, - })); - - await expect(run()).rejects.toMatchObject({ - name: "S2Error", - message: "Bad Gateway", - status: 502, - }); - }); - - it("wraps thrown errors as S2Error via s2Error()", async () => { - const run = () => - withS2Error(async () => { - throw new Error("boom"); - }); - - const err = await run().catch((e) => e as S2Error); - expect(err).toBeInstanceOf(S2Error); - expect(err.message).toBe("boom"); - // Generic thrown errors get status 0 in s2Error() - expect(err.status).toBe(0); - }); + it("returns result when response has no error", async () => { + const value = await withS2Error(async () => ({ + data: { ok: 1 }, + error: undefined, + response: { status: 200, statusText: "OK" }, + })); + + expect(value).toMatchObject({ data: { ok: 1 }, response: { status: 200 } }); + }); + + it("throws S2Error with message/code/status when response.error has message", async () => { + const run = () => + withS2Error(async () => ({ + data: undefined, + error: { message: "Bad things", code: "BAD_THING" }, + response: { status: 400, statusText: "Bad Request" }, + })); + + await expect(run()).rejects.toMatchObject({ + name: "S2Error", + message: "Bad things", + code: "BAD_THING", + status: 400, + }); + }); + + it("falls back to HTTP statusText when error lacks message", async () => { + const run = () => + withS2Error(async () => ({ + data: undefined, + error: { something: "else" }, + response: { status: 502, statusText: "Bad Gateway" }, + })); + + await expect(run()).rejects.toMatchObject({ + name: "S2Error", + message: "Bad Gateway", + status: 502, + }); + }); + + it("wraps thrown errors as S2Error via s2Error()", async () => { + const run = () => + withS2Error(async () => { + throw new Error("boom"); + }); + + const err = await run().catch((e) => e as S2Error); + expect(err).toBeInstanceOf(S2Error); + expect(err.message).toBe("boom"); + // Generic thrown errors get status 0 in s2Error() + expect(err.status).toBe(0); + }); }); - From e248c0ac3081d99411dfc862f14b6c0d3cf566b1 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 19:23:30 -0800 Subject: [PATCH 18/26] error reform --- src/batch-transform.ts | 2 + src/error.ts | 83 ++++++++++++++++++++---- src/lib/retry.ts | 18 ++--- src/lib/stream/transport/fetch/index.ts | 2 + src/lib/stream/transport/fetch/shared.ts | 3 + src/lib/stream/transport/s2s/index.ts | 7 ++ src/s2.ts | 1 + 7 files changed, 94 insertions(+), 22 deletions(-) diff --git a/src/batch-transform.ts b/src/batch-transform.ts index d1c8d80..8cb8439 100644 --- a/src/batch-transform.ts +++ b/src/batch-transform.ts @@ -101,6 +101,8 @@ export class BatchTransform extends TransformStream { if (recordSize > this.maxBatchBytes) { throw new S2Error({ message: `Record size ${recordSize} bytes exceeds maximum batch size of ${this.maxBatchBytes} bytes`, + status: 400, + origin: "sdk", }); } diff --git a/src/error.ts b/src/error.ts index 2c14396..2f8abbc 100644 --- a/src/error.ts +++ b/src/error.ts @@ -9,7 +9,6 @@ function isConnectionError(error: unknown): boolean { const cause = (error as any).cause; let code = (error as any).code; - // TODO check if code exists if (cause && typeof cause === "object") { code = cause.code; } @@ -43,8 +42,8 @@ export function s2Error(error: any): S2Error { const code = cause?.code || "NETWORK_ERROR"; return new S2Error({ message: `Connection failed: ${code}`, - // Could add a specific status or property for connection errors - status: 500, // or 0, or a constant + status: 502, // Bad Gateway for upstream/network issues + origin: "sdk", }); } @@ -52,14 +51,16 @@ export function s2Error(error: any): S2Error { if (error instanceof Error && error.name === "AbortError") { return new S2Error({ message: "Request cancelled", - status: undefined, + status: 499, // Client Closed Request (nginx non-standard) + origin: "sdk", }); } // Other unknown errors return new S2Error({ message: error instanceof Error ? error.message : "Unknown error", - status: 0, + status: 0, // Non-HTTP/internal error sentinel + origin: "sdk", }); } @@ -76,7 +77,7 @@ export async function withS2Error(fn: () => Promise): Promise { ) { const err = result.error; if (err) { - const status = result.response?.status as number | undefined; + const status = (result.response?.status as number | undefined) ?? 500; const statusText = result.response?.statusText as string | undefined; // If server provided structured error with message/code, use it @@ -85,6 +86,7 @@ export async function withS2Error(fn: () => Promise): Promise { message: (err as any).message ?? statusText ?? "Error", code: (err as any).code ?? undefined, status, + origin: "server", }); } @@ -92,6 +94,7 @@ export async function withS2Error(fn: () => Promise): Promise { throw new S2Error({ message: statusText ?? "Request failed", status, + origin: "server", }); } } @@ -127,7 +130,7 @@ export async function withS2Data( Object.prototype.hasOwnProperty.call(res, "data") || Object.prototype.hasOwnProperty.call(res, "response")) ) { - const status = res.response?.status as number | undefined; + const status = (res.response?.status as number | undefined) ?? 500; const statusText = res.response?.statusText as string | undefined; if (res.error) { const err = res.error; @@ -136,15 +139,24 @@ export async function withS2Data( message: (err as any).message ?? statusText ?? "Error", code: (err as any).code ?? undefined, status, + origin: "server", }); } - throw new S2Error({ message: statusText ?? "Request failed", status }); + throw new S2Error({ + message: statusText ?? "Request failed", + status, + origin: "server", + }); } // No error if (typeof res.data !== "undefined") return res.data as T; // Treat 204 as success for void endpoints if (status === 204) return undefined as T; - throw new S2Error({ message: "Empty response", status }); + throw new S2Error({ + message: "Empty response", + status, + origin: "server", + }); } // Not a generated client response; return as-is return res as T; @@ -162,24 +174,70 @@ export async function withS2Data( */ export class S2Error extends Error { public readonly code?: string; - public readonly status?: number; + public readonly status: number; + /** Optional structured error details for diagnostics. */ + public readonly data?: unknown; + /** Origin of the error: server (HTTP response) or sdk (local). */ + public readonly origin: "server" | "sdk"; constructor({ message, code, status, + data, + origin, }: { message: string; code?: string; status?: number; + data?: unknown; + origin?: "server" | "sdk"; }) { super(message); this.code = code; - this.status = status; + // Ensure status is always a number (0 for non-HTTP/internal errors) + this.status = typeof status === "number" ? status : 0; + this.data = data; + this.origin = origin ?? "sdk"; this.name = "S2Error"; } } +/** Helper: construct a non-retryable invariant violation error (400). */ +export function invariantViolation( + message: string, + details?: unknown, +): S2Error { + return new S2Error({ + message: `Invariant violation: ${message}`, + code: "INTERNAL_ERROR", + status: 500, + origin: "sdk", + data: details, + }); +} + +/** Helper: construct an internal SDK error (status 0, never retried). */ +export function internalSdkError(message: string, details?: unknown): S2Error { + return new S2Error({ + message: `Internal SDK error: ${message}`, + code: "INTERNAL_SDK_ERROR", + status: 0, + origin: "sdk", + data: details, + }); +} + +/** Helper: construct an aborted/cancelled error (499). */ +export function abortedError(message: string = "Request cancelled"): S2Error { + return new S2Error({ + message, + code: "ABORTED", + status: 499, + origin: "sdk", + }); +} + /** * Thrown when an append operation fails due to a sequence number mismatch. * @@ -208,6 +266,7 @@ export class SeqNumMismatchError extends S2Error { message: `${message}\nExpected sequence number: ${expectedSeqNum}`, code, status, + origin: "server", }); this.name = "SeqNumMismatchError"; this.expectedSeqNum = expectedSeqNum; @@ -242,6 +301,7 @@ export class FencingTokenMismatchError extends S2Error { message: `${message}\nExpected fencing token: ${expectedFencingToken}`, code, status, + origin: "server", }); this.name = "FencingTokenMismatchError"; this.expectedFencingToken = expectedFencingToken; @@ -271,6 +331,7 @@ export class RangeNotSatisfiableError extends S2Error { message, code, status, + origin: "server", }); this.name = "RangeNotSatisfiableError"; } diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 8c4547d..f5ba182 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -1,6 +1,6 @@ import createDebug from "debug"; import type { RetryConfig } from "../common.js"; -import { S2Error, s2Error, withS2Error } from "../error.js"; +import { invariantViolation, S2Error, s2Error, withS2Error } from "../error.js"; import type { AppendAck, StreamPosition } from "../generated/index.js"; import { meteredSizeBytes } from "../utils.js"; import type { AppendResult, CloseResult } from "./result.js"; @@ -787,11 +787,9 @@ export class AppendSession implements AsyncDisposable { // Invariant check: ack count matches batch count const ackCount = Number(ack.end.seq_num) - Number(ack.start.seq_num); if (ackCount !== head.expectedCount) { - const error = new S2Error({ - message: `Ack count mismatch: expected ${head.expectedCount}, got ${ackCount}`, - status: 500, - code: "INTERNAL_ERROR", - }); + const error = invariantViolation( + `Ack count mismatch: expected ${head.expectedCount}, got ${ackCount}`, + ); debug("invariant violation: %s", error.message); await this.abort(error); return; @@ -802,11 +800,9 @@ export class AppendSession implements AsyncDisposable { const prevEnd = BigInt(this._lastAckedPosition.end.seq_num); const currentEnd = BigInt(ack.end.seq_num); if (currentEnd <= prevEnd) { - const error = new S2Error({ - message: `Sequence number not strictly increasing: previous=${prevEnd}, current=${currentEnd}`, - status: 500, - code: "INTERNAL_ERROR", - }); + const error = invariantViolation( + `Sequence number not strictly increasing: previous=${prevEnd}, current=${currentEnd}`, + ); debug("invariant violation: %s", error.message); await this.abort(error); return; diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index fa23503..2878402 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -84,6 +84,8 @@ export class FetchReadSession if (!response.response.body) { const error = new S2Error({ message: "No body in SSE response", + status: 502, + origin: "sdk", }); return FetchReadSession.createErrorSession(error); } diff --git a/src/lib/stream/transport/fetch/shared.ts b/src/lib/stream/transport/fetch/shared.ts index 1744ede..0d1e892 100644 --- a/src/lib/stream/transport/fetch/shared.ts +++ b/src/lib/stream/transport/fetch/shared.ts @@ -56,6 +56,7 @@ export async function streamRead( message: response.error.message, code: response.error.code ?? undefined, status: response.response.status, + origin: "server", }); } else { // special case for 416 - Range Not Satisfiable @@ -193,6 +194,7 @@ export async function streamAppend( message: response.error.message, code: response.error.code ?? undefined, status: response.response.status, + origin: "server", }); } else { // special case for 412 - append condition failed @@ -215,6 +217,7 @@ export async function streamAppend( throw new S2Error({ message: "Append condition failed", status: response.response.status, + origin: "server", }); } } diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 7481b74..2364b1a 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -372,6 +372,7 @@ class S2SReadSession message: errorJson.message ?? "Unknown error", code: errorJson.code, status: responseCode, + origin: "server", }), ); } catch { @@ -379,6 +380,7 @@ class S2SReadSession new S2Error({ message: errorText || "Unknown error", status: responseCode, + origin: "server", }), ); } @@ -459,6 +461,8 @@ class S2SReadSession safeError( new S2Error({ message: `Failed to parse ReadBatch: ${err}`, + status: 500, + origin: "sdk", }), ); } @@ -473,6 +477,7 @@ class S2SReadSession : new S2Error({ message: `Failed to process read data: ${error}`, status: 500, + origin: "sdk", }), ); } @@ -486,6 +491,7 @@ class S2SReadSession message: `Stream ended with error: ${stream.rstCode}`, status: 500, code: "stream reset", + origin: "sdk", }), ); } @@ -498,6 +504,7 @@ class S2SReadSession message: "Stream closed with unparsed data remaining", status: 500, code: "STREAM_CLOSED_PREMATURELY", + origin: "sdk", }), ); } else { diff --git a/src/s2.ts b/src/s2.ts index 1c1acb6..d2af8ec 100644 --- a/src/s2.ts +++ b/src/s2.ts @@ -55,6 +55,7 @@ export class S2 { message: err instanceof Error ? err.message : "Unknown error", code: res.statusText, status: res.status, + origin: "server", }); }); From f3ace6fde69e774ecd8baf90166e17a1e22321c4 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 20:07:37 -0800 Subject: [PATCH 19/26] a --- src/basins.ts | 2 +- src/batch-transform.ts | 4 - src/error.ts | 69 ++++++++++++ src/lib/retry.ts | 38 +++++-- src/lib/stream/transport/fetch/index.ts | 11 +- src/lib/stream/transport/fetch/shared.ts | 138 ++++++++++------------- src/lib/stream/transport/s2s/index.ts | 86 ++++++-------- src/lib/stream/types.ts | 1 - src/metrics.ts | 2 +- src/s2.ts | 1 - src/streams.ts | 2 +- src/tests/appendSession.e2e.test.ts | 14 +-- src/tests/readSession.e2e.test.ts | 14 +-- 13 files changed, 209 insertions(+), 173 deletions(-) diff --git a/src/basins.ts b/src/basins.ts index 44ad08f..37076eb 100644 --- a/src/basins.ts +++ b/src/basins.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Data } from "./error.js"; +import { withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type BasinConfig, diff --git a/src/batch-transform.ts b/src/batch-transform.ts index 8cb8439..2be3165 100644 --- a/src/batch-transform.ts +++ b/src/batch-transform.ts @@ -1,9 +1,6 @@ -import createDebug from "debug"; import { S2Error } from "./error.js"; import { AppendRecord, meteredSizeBytes } from "./utils.js"; -const debug = createDebug("s2:batch_transform"); - export interface BatchTransformArgs { /** Duration in milliseconds to wait before flushing a batch (default: 5ms) */ lingerDurationMillis?: number; @@ -162,7 +159,6 @@ export class BatchTransform extends TransformStream { if (match_seq_num !== undefined) { batch.match_seq_num = match_seq_num; } - debug({ batch }); this.controller.enqueue(batch); } diff --git a/src/error.ts b/src/error.ts index 2f8abbc..cf1a6f1 100644 --- a/src/error.ts +++ b/src/error.ts @@ -336,3 +336,72 @@ export class RangeNotSatisfiableError extends S2Error { this.name = "RangeNotSatisfiableError"; } } + +/** + * Build a generic S2Error from HTTP status and optional payload. + * If the payload contains a structured { message, code }, those are preferred. + */ +export function makeServerError( + response: { status?: number; statusText?: string }, + payload?: unknown, +): S2Error { + const status = typeof response.status === "number" ? response.status : 500; + // Pull message/code from structured payload when present + if (payload && typeof payload === "object" && "message" in (payload as any)) { + return new S2Error({ + message: (payload as any).message ?? response.statusText ?? "Error", + code: (payload as any).code ?? undefined, + status, + origin: "server", + }); + } + // Fallbacks + let message: string | undefined = undefined; + if (typeof payload === "string" && payload.trim().length > 0) { + message = payload; + } + return new S2Error({ + message: message ?? response.statusText ?? "Request failed", + status, + origin: "server", + }); +} + +/** Map 412 Precondition Failed append errors to rich error types. */ +export function makeAppendPreconditionError( + status: number, + json: any, +): S2Error { + if (json && typeof json === "object") { + if ("seq_num_mismatch" in json) { + const expected = Number(json.seq_num_mismatch); + return new SeqNumMismatchError({ + message: "Append condition failed: sequence number mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedSeqNum: expected, + }); + } + if ("fencing_token_mismatch" in json) { + const expected = String(json.fencing_token_mismatch); + return new FencingTokenMismatchError({ + message: "Append condition failed: fencing token mismatch", + code: "APPEND_CONDITION_FAILED", + status, + expectedFencingToken: expected, + }); + } + if ("message" in json) { + return new S2Error({ + message: json.message ?? "Append condition failed", + status, + origin: "server", + }); + } + } + return new S2Error({ + message: "Append condition failed", + status, + origin: "server", + }); +} diff --git a/src/lib/retry.ts b/src/lib/retry.ts index f5ba182..50ec9d3 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -245,6 +245,11 @@ export class ReadSession< baselineWait - (elapsedSeconds + delay / 1000), ) as any; } + // Proactively cancel the current transport session before retrying + try { + await session.cancel?.("retry"); + } catch {} + debug("will retry after %dms, status=%s", delay, error.status); await sleep(delay); attempt++; @@ -257,7 +262,7 @@ export class ReadSession< return; } - // Success: enqueue the record (don't reset attempt counter - track session attempts) + // Success: enqueue the record and reset retry attempt counter const record = result.value; this._nextReadPosition = { seq_num: record.seq_num + 1, @@ -375,7 +380,7 @@ type InflightEntry = { args?: Omit & { precalculatedSize?: number }; expectedCount: number; meteredBytes: number; - enqueuedAt: number; // Timestamp for timeout anchoring + attemptStartedMonotonicMs?: number; // Monotonic timestamp (performance.now) for per-attempt ack timeout anchoring innerPromise: Promise; // Promise from transport session maybeResolve?: (result: AppendResult) => void; // Resolver for submit() callers }; @@ -565,7 +570,6 @@ export class AppendSession implements AsyncDisposable { args, expectedCount: records.length, meteredBytes: batchMeteredSize, - enqueuedAt: Date.now(), innerPromise: new Promise(() => {}), // Never-resolving placeholder maybeResolve: resolve, __needsSubmit: true, // Mark for pump to submit @@ -729,10 +733,9 @@ export class AppendSession implements AsyncDisposable { // Get head entry (we know it exists because we checked length above) const head = this.inflight[0]!; debug( - "[PUMP] processing head: expectedCount=%d, meteredBytes=%d, enqueuedAt=%d", + "[PUMP] processing head: expectedCount=%d, meteredBytes=%d", head.expectedCount, head.meteredBytes, - head.enqueuedAt, ); // Ensure session exists @@ -753,6 +756,7 @@ export class AppendSession implements AsyncDisposable { entry.expectedCount, entry.meteredBytes, ); + entry.attemptStartedMonotonicMs = performance.now(); entry.innerPromise = this.session.submit(entry.records, entry.args); delete (entry as any).__needsSubmit; } @@ -764,10 +768,13 @@ export class AppendSession implements AsyncDisposable { debug("[PUMP] got result: kind=%s", result.kind); if (result.kind === "timeout") { - // Ack timeout - fatal - const elapsed = Date.now() - head.enqueuedAt; + // Ack timeout - fatal (per-attempt) + const attemptElapsed = + head.attemptStartedMonotonicMs != null + ? Math.round(performance.now() - head.attemptStartedMonotonicMs) + : undefined; const error = new S2Error({ - message: `Request timeout after ${elapsed}ms (${head.expectedCount} records, ${head.meteredBytes} bytes, enqueued at ${new Date(head.enqueuedAt).toISOString()})`, + message: `Request timeout after ${attemptElapsed ?? "unknown"}ms (${head.expectedCount} records, ${head.meteredBytes} bytes)`, status: 408, code: "REQUEST_TIMEOUT", }); @@ -894,12 +901,20 @@ export class AppendSession implements AsyncDisposable { /** * Wait for head entry's innerPromise with timeout. * Returns either the settled result or a timeout indicator. + * + * Per-attempt ack timeout semantics: + * - The deadline is computed from the most recent (re)submit attempt using + * a monotonic clock (performance.now) to avoid issues with wall clock + * adjustments. + * - If attempt start is missing (for backward compatibility), we measure + * from "now" with the full timeout window. */ private async waitForHead( head: InflightEntry, ): Promise<{ kind: "settled"; value: AppendResult } | { kind: "timeout" }> { - const deadline = head.enqueuedAt + this.requestTimeoutMillis; - const remaining = Math.max(0, deadline - Date.now()); + const startMono = head.attemptStartedMonotonicMs ?? performance.now(); + const deadline = startMono + this.requestTimeoutMillis; + const remaining = Math.max(0, deadline - performance.now()); let timer: any; const timeoutP = new Promise<{ kind: "timeout" }>((resolve) => { @@ -959,13 +974,14 @@ export class AppendSession implements AsyncDisposable { // Store session in local variable to help TypeScript type narrowing const session: TransportAppendSession = this.session; - // Resubmit all inflight entries (replace their innerPromise) + // Resubmit all inflight entries (replace their innerPromise and reset attempt start) debug("resubmitting %d inflight entries", this.inflight.length); for (const entry of this.inflight) { // Attach .catch to superseded promise to avoid unhandled rejection entry.innerPromise.catch(() => {}); // Create new promise from new session + entry.attemptStartedMonotonicMs = performance.now(); entry.innerPromise = session.submit(entry.records, entry.args); } diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 2878402..82dc015 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -33,6 +33,7 @@ import type { ReadResult, ReadSession, SessionTransport, + TransportAppendSession, TransportConfig, TransportReadSession, } from "../../types.js"; @@ -84,6 +85,7 @@ export class FetchReadSession if (!response.response.body) { const error = new S2Error({ message: "No body in SSE response", + code: "INVALID_RESPONSE", status: 502, origin: "sdk", }); @@ -318,16 +320,15 @@ export class FetchReadSession // Removed AcksStream - transport sessions no longer expose streams /** - * "Dumb" transport session for appending records via HTTP/1.1. + * Fetch-based transport session for appending records via HTTP/1.1. * Queues append requests and ensures only one is in-flight at a time (single-flight). * No backpressure, no retry logic, no streams - just submit/close with value-encoded errors. */ -export class FetchAppendSession { +export class FetchAppendSession implements TransportAppendSession { private queue: Array<{ records: AppendRecord[]; fencing_token?: string; match_seq_num?: number; - meteredSize: number; }> = []; private pendingResolvers: Array<{ resolve: (result: AppendResult) => void; @@ -403,7 +404,6 @@ export class FetchAppendSession { match_seq_num?: number; precalculatedSize?: number; }, - precalculatedSize?: number, ): Promise { // Validate closed state if (this.closed) { @@ -428,7 +428,7 @@ export class FetchAppendSession { } // Validate metered size (use precalculated if provided) - let batchMeteredSize = precalculatedSize ?? args?.precalculatedSize ?? 0; + let batchMeteredSize = args?.precalculatedSize ?? 0; if (batchMeteredSize === 0) { for (const record of recordsArray) { batchMeteredSize += meteredSizeBytes(record); @@ -452,7 +452,6 @@ export class FetchAppendSession { records: recordsArray, fencing_token: args?.fencing_token, match_seq_num: args?.match_seq_num, - meteredSize: batchMeteredSize, }); this.pendingResolvers.push({ resolve }); diff --git a/src/lib/stream/transport/fetch/shared.ts b/src/lib/stream/transport/fetch/shared.ts index 0d1e892..d852b98 100644 --- a/src/lib/stream/transport/fetch/shared.ts +++ b/src/lib/stream/transport/fetch/shared.ts @@ -1,9 +1,10 @@ import type { S2RequestOptions } from "../../../../common.js"; import { - FencingTokenMismatchError, + makeAppendPreconditionError, + makeServerError, RangeNotSatisfiableError, S2Error, - SeqNumMismatchError, + s2Error, } from "../../../../error.js"; import type { Client } from "../../../../generated/client/index.js"; import { @@ -39,31 +40,31 @@ export async function streamRead( options?: S2RequestOptions, ) { const { as, ...queryParams } = args ?? {}; - const response = await read({ - client, - path: { - stream, - }, - headers: { - ...(as === "bytes" ? { "s2-format": "base64" } : {}), - }, - query: queryParams, - ...options, - }); + let response: any; + try { + response = await read({ + client, + path: { + stream, + }, + headers: { + ...(as === "bytes" ? { "s2-format": "base64" } : {}), + }, + query: queryParams, + ...options, + }); + } catch (error) { + throw s2Error(error); + } if (response.error) { - if ("message" in response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - origin: "server", - }); - } else { - // special case for 416 - Range Not Satisfiable - throw new RangeNotSatisfiableError({ - status: response.response.status, - }); + const status = response.response.status; + if (status === 416) { + throw new RangeNotSatisfiableError({ status }); } + throw makeServerError( + { status, statusText: response.response.statusText }, + response.error, + ); } if (args?.as === "bytes") { @@ -87,12 +88,14 @@ export async function streamRead( } else { const res: ReadBatch<"string"> = { ...response.data, - records: response.data.records?.map((record) => ({ - ...record, - headers: record.headers - ? Object.fromEntries(record.headers) - : undefined, - })), + records: response.data.records?.map( + (record: GeneratedSequencedRecord) => ({ + ...record, + headers: record.headers + ? Object.fromEntries(record.headers) + : undefined, + }), + ), }; return res as ReadBatch; } @@ -173,54 +176,35 @@ export async function streamAppend( } } - const response = await append({ - client, - path: { - stream, - }, - body: { - fencing_token: args?.fencing_token, - match_seq_num: args?.match_seq_num, - records: encodedRecords, - }, - headers: { - ...(hasAnyBytesRecords ? { "s2-format": "base64" } : {}), - }, - ...options, - }); + let response: any; + try { + response = await append({ + client, + path: { + stream, + }, + body: { + fencing_token: args?.fencing_token, + match_seq_num: args?.match_seq_num, + records: encodedRecords, + }, + headers: { + ...(hasAnyBytesRecords ? { "s2-format": "base64" } : {}), + }, + ...options, + }); + } catch (error) { + throw s2Error(error); + } if (response.error) { - if ("message" in response.error) { - throw new S2Error({ - message: response.error.message, - code: response.error.code ?? undefined, - status: response.response.status, - origin: "server", - }); - } else { - // special case for 412 - append condition failed - if ("seq_num_mismatch" in response.error) { - throw new SeqNumMismatchError({ - message: "Append condition failed: sequence number mismatch", - code: "APPEND_CONDITION_FAILED", - status: response.response.status, - expectedSeqNum: response.error.seq_num_mismatch, - }); - } else if ("fencing_token_mismatch" in response.error) { - throw new FencingTokenMismatchError({ - message: "Append condition failed: fencing token mismatch", - code: "APPEND_CONDITION_FAILED", - status: response.response.status, - expectedFencingToken: response.error.fencing_token_mismatch, - }); - } else { - // fallback for unknown 412 error format - throw new S2Error({ - message: "Append condition failed", - status: response.response.status, - origin: "server", - }); - } + const status = response.response.status; + if (status === 412) { + throw makeAppendPreconditionError(status, response.error); } + throw makeServerError( + { status, statusText: response.response.statusText }, + response.error, + ); } return response.data; } diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 2364b1a..8a50611 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -9,10 +9,10 @@ import * as http2 from "node:http2"; import createDebug from "debug"; import type { S2RequestOptions } from "../../../../common.js"; import { - FencingTokenMismatchError, + makeAppendPreconditionError, + makeServerError, RangeNotSatisfiableError, S2Error, - SeqNumMismatchError, } from "../../../../error.js"; import type { AppendAck, StreamPosition } from "../../../../generated/index.js"; import { @@ -39,6 +39,7 @@ import type { ReadResult, ReadSession, SessionTransport, + TransportAppendSession, TransportConfig, TransportReadSession, } from "../../types.js"; @@ -397,27 +398,27 @@ class S2SReadSession try { const errorJson = JSON.parse(errorText); const status = frame.statusCode ?? 500; - const message = errorJson.message ?? "Unknown error"; - const code = errorJson.code; // Map known read errors if (status === 416) { safeError(new RangeNotSatisfiableError({ status })); } else { safeError( - new S2Error({ - message, - code, - status, - }), + makeServerError( + { status, statusText: undefined }, + errorJson, + ), ); } } catch { safeError( - new S2Error({ - message: errorText || "Unknown error", - status: frame.statusCode, - }), + makeServerError( + { + status: frame.statusCode ?? 500, + statusText: undefined, + }, + errorText, + ), ); } } else { @@ -619,11 +620,11 @@ class S2SReadSession // Removed S2SAcksStream - transport sessions no longer expose streams /** - * "Dumb" transport session for appending records via HTTP/2. + * Fetch-based transport session for appending records via HTTP/2. * Pipelined: multiple requests can be in-flight simultaneously. * No backpressure, no retry logic, no streams - just submit/close with value-encoded errors. */ -class S2SAppendSession { +class S2SAppendSession implements TransportAppendSession { private http2Stream?: http2.ClientHttp2Stream; private parser = new S2SFrameParser(); private closed = false; @@ -662,7 +663,7 @@ class S2SAppendSession { sessionOptions?: AppendSessionOptions, private options?: S2RequestOptions, ) { - // No stream setup - transport is "dumb" + // No stream setup // Initialization happens lazily on first submit } @@ -719,51 +720,28 @@ class S2SAppendSession { const status = frame.statusCode ?? 500; try { const errorJson = JSON.parse(errorText); - const message = errorJson.message ?? "Unknown error"; - const code = errorJson.code; - - // Map known append errors (412 Precondition Failed) if (status === 412) { - if ("seq_num_mismatch" in errorJson) { - const expected = Number(errorJson.seq_num_mismatch); - queueMicrotask(() => - safeError( - new SeqNumMismatchError({ - message: - "Append condition failed: sequence number mismatch", - code: "APPEND_CONDITION_FAILED", - status, - expectedSeqNum: expected, - }), - ), - ); - } else if ("fencing_token_mismatch" in errorJson) { - const expected = String(errorJson.fencing_token_mismatch); - queueMicrotask(() => - safeError( - new FencingTokenMismatchError({ - message: - "Append condition failed: fencing token mismatch", - code: "APPEND_CONDITION_FAILED", - status, - expectedFencingToken: expected, - }), - ), - ); - } else { - queueMicrotask(() => - safeError(new S2Error({ message, code, status })), - ); - } + queueMicrotask(() => + safeError(makeAppendPreconditionError(status, errorJson)), + ); } else { queueMicrotask(() => - safeError(new S2Error({ message, code, status })), + safeError( + makeServerError( + { status, statusText: undefined }, + errorJson, + ), + ), ); } } catch { - const message = errorText || "Unknown error"; queueMicrotask(() => - safeError(new S2Error({ message, status })), + safeError( + makeServerError( + { status, statusText: undefined }, + errorText, + ), + ), ); } } diff --git a/src/lib/stream/types.ts b/src/lib/stream/types.ts index 3ac9236..1a735a8 100644 --- a/src/lib/stream/types.ts +++ b/src/lib/stream/types.ts @@ -64,7 +64,6 @@ export interface AcksStream AsyncIterable {} /** - * Transport-facing interface for "dumb" append sessions. * Transports only implement submit/close with value-encoded errors (discriminated unions). * No backpressure, no retry, no streams - AppendSession adds those. */ diff --git a/src/metrics.ts b/src/metrics.ts index c25f063..cae5fc1 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Data } from "./error.js"; +import { withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type AccountMetricsData, diff --git a/src/s2.ts b/src/s2.ts index d2af8ec..1e305b0 100644 --- a/src/s2.ts +++ b/src/s2.ts @@ -53,7 +53,6 @@ export class S2 { this.client.interceptors.error.use((err, res, req, opt) => { return new S2Error({ message: err instanceof Error ? err.message : "Unknown error", - code: res.statusText, status: res.status, origin: "server", }); diff --git a/src/streams.ts b/src/streams.ts index 637499a..06534da 100644 --- a/src/streams.ts +++ b/src/streams.ts @@ -1,5 +1,5 @@ import type { DataToObject, RetryConfig, S2RequestOptions } from "./common.js"; -import { S2Error, withS2Data } from "./error.js"; +import { withS2Data } from "./error.js"; import type { Client } from "./generated/client/types.gen.js"; import { type CreateStreamData, diff --git a/src/tests/appendSession.e2e.test.ts b/src/tests/appendSession.e2e.test.ts index ed3bd13..c6b7762 100644 --- a/src/tests/appendSession.e2e.test.ts +++ b/src/tests/appendSession.e2e.test.ts @@ -3,8 +3,10 @@ import { AppendRecord, S2 } from "../index.js"; import type { SessionTransports } from "../lib/stream/types.js"; const transports: SessionTransports[] = ["fetch", "s2s"]; +const hasEnv = !!process.env.S2_ACCESS_TOKEN && !!process.env.S2_BASIN; +const describeIf = hasEnv ? describe : describe.skip; -describe("AppendSession Integration Tests", () => { +describeIf("AppendSession Integration Tests", () => { let s2: S2; let basinName: string; let streamName: string; @@ -12,13 +14,9 @@ describe("AppendSession Integration Tests", () => { beforeAll(() => { const token = process.env.S2_ACCESS_TOKEN; const basin = process.env.S2_BASIN; - if (!token || !basin) { - throw new Error( - "S2_ACCESS_TOKEN and S2_BASIN environment variables are required for e2e tests", - ); - } - s2 = new S2({ accessToken: token }); - basinName = basin; + if (!token || !basin) return; + s2 = new S2({ accessToken: token! }); + basinName = basin!; }); beforeAll(async () => { diff --git a/src/tests/readSession.e2e.test.ts b/src/tests/readSession.e2e.test.ts index 5ef4cae..dd6b2e4 100644 --- a/src/tests/readSession.e2e.test.ts +++ b/src/tests/readSession.e2e.test.ts @@ -3,8 +3,10 @@ import { AppendRecord, S2 } from "../index.js"; import type { SessionTransports } from "../lib/stream/types.js"; const transports: SessionTransports[] = ["fetch", "s2s"]; +const hasEnv = !!process.env.S2_ACCESS_TOKEN && !!process.env.S2_BASIN; +const describeIf = hasEnv ? describe : describe.skip; -describe("ReadSession Integration Tests", () => { +describeIf("ReadSession Integration Tests", () => { let s2: S2; let basinName: string; let streamName: string; @@ -12,13 +14,9 @@ describe("ReadSession Integration Tests", () => { beforeAll(() => { const token = process.env.S2_ACCESS_TOKEN; const basin = process.env.S2_BASIN; - if (!token || !basin) { - throw new Error( - "S2_ACCESS_TOKEN and S2_BASIN environment variables are required for e2e tests", - ); - } - s2 = new S2({ accessToken: token }); - basinName = basin; + if (!token || !basin) return; + s2 = new S2({ accessToken: token! }); + basinName = basin!; }); beforeAll(async () => { From 9cafc84eb6a142c1b41734977e81a4d736b85f2d Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 23:41:40 -0800 Subject: [PATCH 20/26] better debug logging --- src/lib/retry.ts | 122 +++++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/src/lib/retry.ts b/src/lib/retry.ts index 50ec9d3..43eba13 100644 --- a/src/lib/retry.ts +++ b/src/lib/retry.ts @@ -16,7 +16,9 @@ import type { TransportReadSession, } from "./stream/types.js"; -const debug = createDebug("s2:retry"); +const debugWith = createDebug("s2:retry:with"); +const debugRead = createDebug("s2:retry:read"); +const debugSession = createDebug("s2:retry:session"); /** * Default retry configuration. @@ -97,7 +99,7 @@ export async function withRetries( // If maxAttempts is 0, don't retry at all if (config.maxAttempts === 0) { - debug("maxAttempts is 0, retries disabled"); + debugWith("maxAttempts is 0, retries disabled"); return fn(); } @@ -107,13 +109,13 @@ export async function withRetries( try { const result = await fn(); if (attempt > 0) { - debug("succeeded after %d retries", attempt); + debugWith("succeeded after %d retries", attempt); } return result; } catch (error) { // withRetry only handles S2Errors (withS2Error should be called first) if (!(error instanceof S2Error)) { - debug("non-S2Error thrown, rethrowing immediately: %s", error); + debugWith("non-S2Error thrown, rethrowing immediately: %s", error); throw error; } @@ -121,19 +123,19 @@ export async function withRetries( // Don't retry if this is the last attempt if (attempt === config.maxAttempts) { - debug("max attempts exhausted, throwing error"); + debugWith("max attempts exhausted, throwing error"); break; } // Check if error is retryable if (!isPolicyCompliant(config, lastError) || !isRetryable(lastError)) { - debug("error not retryable, throwing immediately"); + debugWith("error not retryable, throwing immediately"); throw error; } // Calculate delay and wait before retrying const delay = calculateDelay(attempt, config.retryBackoffDurationMs); - debug( + debugWith( "retryable error, backing off for %dms, status=%s", delay, error.status, @@ -186,7 +188,7 @@ export class ReadSession< let attempt = 0; while (true) { - debug("starting read session with args: %o", nextArgs); + debugRead("starting read session with args: %o", nextArgs); session = await generator(nextArgs); const reader = session.getReader(); @@ -250,14 +252,18 @@ export class ReadSession< await session.cancel?.("retry"); } catch {} - debug("will retry after %dms, status=%s", delay, error.status); + debugRead( + "will retry after %dms, status=%s", + delay, + error.status, + ); await sleep(delay); attempt++; break; // Break inner loop to retry } // Error is not retryable or attempts exhausted - debug("error in retry loop: %s", error); + debugRead("error in retry loop: %s", error); controller.error(error); return; } @@ -555,7 +561,7 @@ export class AppendSession implements AsyncDisposable { // Check for fatal error (e.g., from abort()) if (this.fatalError) { - debug( + debugSession( "[SUBMIT] rejecting due to fatal error: %s", this.fatalError.message, ); @@ -575,7 +581,7 @@ export class AppendSession implements AsyncDisposable { __needsSubmit: true, // Mark for pump to submit }; - debug( + debugSession( "[SUBMIT] enqueueing %d records (%d bytes): inflight=%d->%d, queuedBytes=%d->%d", records.length, batchMeteredSize, @@ -602,7 +608,7 @@ export class AppendSession implements AsyncDisposable { * Wait for capacity before allowing write to proceed (writable only). */ private async waitForCapacity(bytes: number): Promise { - debug( + debugSession( "[CAPACITY] checking for %d bytes: queuedBytes=%d, pendingBytes=%d, maxQueuedBytes=%d, inflight=%d", bytes, this.queuedBytes, @@ -615,7 +621,7 @@ export class AppendSession implements AsyncDisposable { while (true) { // Check for fatal error before adding to pendingBytes if (this.fatalError) { - debug( + debugSession( "[CAPACITY] fatal error detected, rejecting: %s", this.fatalError.message, ); @@ -629,7 +635,7 @@ export class AppendSession implements AsyncDisposable { this.maxInflightBatches === undefined || this.inflight.length < this.maxInflightBatches ) { - debug( + debugSession( "[CAPACITY] capacity available, adding %d to pendingBytes", bytes, ); @@ -640,11 +646,11 @@ export class AppendSession implements AsyncDisposable { // No capacity - wait // WritableStream enforces writer lock, so only one write can be blocked at a time - debug("[CAPACITY] no capacity, waiting for release"); + debugSession("[CAPACITY] no capacity, waiting for release"); await new Promise((resolve) => { this.capacityWaiter = resolve; }); - debug("[CAPACITY] woke up, rechecking"); + debugSession("[CAPACITY] woke up, rechecking"); } } @@ -652,7 +658,7 @@ export class AppendSession implements AsyncDisposable { * Release capacity and wake waiter if present. */ private releaseCapacity(bytes: number): void { - debug( + debugSession( "[CAPACITY] releasing %d bytes: queuedBytes=%d->%d, pendingBytes=%d->%d, hasWaiter=%s", bytes, this.queuedBytes, @@ -667,7 +673,7 @@ export class AppendSession implements AsyncDisposable { // Wake single waiter const waiter = this.capacityWaiter; if (waiter) { - debug("[CAPACITY] waking waiter"); + debugSession("[CAPACITY] waking waiter"); this.capacityWaiter = undefined; waiter(); } @@ -682,7 +688,7 @@ export class AppendSession implements AsyncDisposable { } this.pumpPromise = this.runPump().catch((e) => { - debug("pump crashed unexpectedly: %s", e); + debugSession("pump crashed unexpectedly: %s", e); // This should never happen - pump handles all errors internally }); } @@ -691,10 +697,10 @@ export class AppendSession implements AsyncDisposable { * Main pump loop: processes inflight queue, handles acks, retries, and recovery. */ private async runPump(): Promise { - debug("pump started"); + debugSession("pump started"); while (true) { - debug( + debugSession( "[PUMP] loop: inflight=%d, queuedBytes=%d, pendingBytes=%d, closing=%s, pumpStopped=%s", this.inflight.length, this.queuedBytes, @@ -705,20 +711,20 @@ export class AppendSession implements AsyncDisposable { // Check if we should stop if (this.pumpStopped) { - debug("[PUMP] stopped by flag"); + debugSession("[PUMP] stopped by flag"); return; } // If closing and queue is empty, stop if (this.closing && this.inflight.length === 0) { - debug("[PUMP] closing and queue empty, stopping"); + debugSession("[PUMP] closing and queue empty, stopping"); this.pumpStopped = true; return; } // If no entries, sleep and continue if (this.inflight.length === 0) { - debug("[PUMP] no entries, sleeping 10ms"); + debugSession("[PUMP] no entries, sleeping 10ms"); // Use interruptible sleep - can be woken by new submissions await Promise.race([ sleep(10), @@ -732,18 +738,18 @@ export class AppendSession implements AsyncDisposable { // Get head entry (we know it exists because we checked length above) const head = this.inflight[0]!; - debug( + debugSession( "[PUMP] processing head: expectedCount=%d, meteredBytes=%d", head.expectedCount, head.meteredBytes, ); // Ensure session exists - debug("[PUMP] ensuring session exists"); + debugSession("[PUMP] ensuring session exists"); await this.ensureSession(); if (!this.session) { // Session creation failed - will retry - debug("[PUMP] session creation failed, sleeping 100ms"); + debugSession("[PUMP] session creation failed, sleeping 100ms"); await sleep(100); continue; } @@ -751,7 +757,7 @@ export class AppendSession implements AsyncDisposable { // Submit ALL entries that need submitting (enables HTTP/2 pipelining for S2S) for (const entry of this.inflight) { if (!entry.innerPromise || (entry as any).__needsSubmit) { - debug( + debugSession( "[PUMP] submitting entry to inner session (%d records, %d bytes)", entry.expectedCount, entry.meteredBytes, @@ -763,9 +769,9 @@ export class AppendSession implements AsyncDisposable { } // Wait for head with timeout - debug("[PUMP] waiting for head result"); + debugSession("[PUMP] waiting for head result"); const result = await this.waitForHead(head); - debug("[PUMP] got result: kind=%s", result.kind); + debugSession("[PUMP] got result: kind=%s", result.kind); if (result.kind === "timeout") { // Ack timeout - fatal (per-attempt) @@ -778,7 +784,7 @@ export class AppendSession implements AsyncDisposable { status: 408, code: "REQUEST_TIMEOUT", }); - debug("ack timeout for head entry: %s", error.message); + debugSession("ack timeout for head entry: %s", error.message); await this.abort(error); return; } @@ -789,7 +795,7 @@ export class AppendSession implements AsyncDisposable { if (appendResult.ok) { // Success! const ack = appendResult.value; - debug("[PUMP] success, got ack", { ack }); + debugSession("[PUMP] success, got ack", { ack }); // Invariant check: ack count matches batch count const ackCount = Number(ack.end.seq_num) - Number(ack.start.seq_num); @@ -797,7 +803,7 @@ export class AppendSession implements AsyncDisposable { const error = invariantViolation( `Ack count mismatch: expected ${head.expectedCount}, got ${ackCount}`, ); - debug("invariant violation: %s", error.message); + debugSession("invariant violation: %s", error.message); await this.abort(error); return; } @@ -810,7 +816,7 @@ export class AppendSession implements AsyncDisposable { const error = invariantViolation( `Sequence number not strictly increasing: previous=${prevEnd}, current=${currentEnd}`, ); - debug("invariant violation: %s", error.message); + debugSession("invariant violation: %s", error.message); await this.abort(error); return; } @@ -828,11 +834,11 @@ export class AppendSession implements AsyncDisposable { try { this.acksController?.enqueue(ack); } catch (e) { - debug("failed to enqueue ack: %s", e); + debugSession("failed to enqueue ack: %s", e); } // Remove from inflight and release capacity - debug( + debugSession( "[PUMP] removing head from inflight, releasing %d bytes", head.meteredBytes, ); @@ -845,7 +851,7 @@ export class AppendSession implements AsyncDisposable { } else { // Error result const error = appendResult.error; - debug( + debugSession( "[PUMP] error: status=%s, message=%s", error.status, error.message, @@ -853,7 +859,7 @@ export class AppendSession implements AsyncDisposable { // Check if retryable if (!isRetryable(error)) { - debug("error not retryable, aborting"); + debugSession("error not retryable, aborting"); await this.abort(error); return; } @@ -863,14 +869,14 @@ export class AppendSession implements AsyncDisposable { this.retryConfig.appendRetryPolicy === "noSideEffects" && !this.isIdempotent(head) ) { - debug("error not policy-compliant (noSideEffects), aborting"); + debugSession("error not policy-compliant (noSideEffects), aborting"); await this.abort(error); return; } // Check max attempts if (this.currentAttempt >= this.retryConfig.maxAttempts) { - debug( + debugSession( "max attempts reached (%d), aborting", this.retryConfig.maxAttempts, ); @@ -887,7 +893,7 @@ export class AppendSession implements AsyncDisposable { this.consecutiveFailures++; this.currentAttempt++; - debug( + debugSession( "performing recovery (attempt %d/%d)", this.currentAttempt, this.retryConfig.maxAttempts, @@ -937,14 +943,14 @@ export class AppendSession implements AsyncDisposable { * Recover from transient error: recreate session and resubmit all inflight entries. */ private async recover(): Promise { - debug("starting recovery"); + debugSession("starting recovery"); // Calculate backoff delay const delay = calculateDelay( this.consecutiveFailures - 1, this.retryConfig.retryBackoffDurationMs, ); - debug("backing off for %dms", delay); + debugSession("backing off for %dms", delay); await sleep(delay); // Teardown old session @@ -952,13 +958,13 @@ export class AppendSession implements AsyncDisposable { try { const closeResult = await this.session.close(); if (!closeResult.ok) { - debug( + debugSession( "error closing old session during recovery: %s", closeResult.error.message, ); } } catch (e) { - debug("exception closing old session: %s", e); + debugSession("exception closing old session: %s", e); } this.session = undefined; } @@ -966,7 +972,7 @@ export class AppendSession implements AsyncDisposable { // Create new session await this.ensureSession(); if (!this.session) { - debug("failed to create new session during recovery"); + debugSession("failed to create new session during recovery"); // Will retry on next pump iteration return; } @@ -975,7 +981,7 @@ export class AppendSession implements AsyncDisposable { const session: TransportAppendSession = this.session; // Resubmit all inflight entries (replace their innerPromise and reset attempt start) - debug("resubmitting %d inflight entries", this.inflight.length); + debugSession("resubmitting %d inflight entries", this.inflight.length); for (const entry of this.inflight) { // Attach .catch to superseded promise to avoid unhandled rejection entry.innerPromise.catch(() => {}); @@ -985,7 +991,7 @@ export class AppendSession implements AsyncDisposable { entry.innerPromise = session.submit(entry.records, entry.args); } - debug("recovery complete"); + debugSession("recovery complete"); } /** @@ -1011,7 +1017,7 @@ export class AppendSession implements AsyncDisposable { this.session = await this.generator(this.sessionOptions); } catch (e) { const error = s2Error(e); - debug("failed to create session: %s", error.message); + debugSession("failed to create session: %s", error.message); // Don't set this.session - will retry later } } @@ -1024,7 +1030,7 @@ export class AppendSession implements AsyncDisposable { return; // Already aborted } - debug("aborting session: %s", error.message); + debugSession("aborting session: %s", error.message); this.fatalError = error; this.pumpStopped = true; @@ -1043,7 +1049,7 @@ export class AppendSession implements AsyncDisposable { try { this.acksController?.error(error); } catch (e) { - debug("failed to error acks controller: %s", e); + debugSession("failed to error acks controller: %s", e); } // Wake capacity waiter to unblock any pending writer @@ -1057,7 +1063,7 @@ export class AppendSession implements AsyncDisposable { try { await this.session.close(); } catch (e) { - debug("error closing session during abort: %s", e); + debugSession("error closing session during abort: %s", e); } this.session = undefined; } @@ -1076,7 +1082,7 @@ export class AppendSession implements AsyncDisposable { return; } - debug("close requested"); + debugSession("close requested"); this.closing = true; // Wake pump if it's sleeping so it can check closing flag @@ -1094,10 +1100,10 @@ export class AppendSession implements AsyncDisposable { try { const result = await this.session.close(); if (!result.ok) { - debug("error closing inner session: %s", result.error.message); + debugSession("error closing inner session: %s", result.error.message); } } catch (e) { - debug("exception closing inner session: %s", e); + debugSession("exception closing inner session: %s", e); } this.session = undefined; } @@ -1106,7 +1112,7 @@ export class AppendSession implements AsyncDisposable { try { this.acksController?.close(); } catch (e) { - debug("error closing acks controller: %s", e); + debugSession("error closing acks controller: %s", e); } this.closed = true; @@ -1116,7 +1122,7 @@ export class AppendSession implements AsyncDisposable { throw this.fatalError; } - debug("close complete"); + debugSession("close complete"); } async [Symbol.asyncDispose](): Promise { From f42ee4ca4e1e653bfd31a17a9e8d5f0b57098bae Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Tue, 11 Nov 2025 23:59:57 -0800 Subject: [PATCH 21/26] a --- src/batch-transform.ts | 39 ++++++++++++++++++++++++++----- src/tests/batch-transform.test.ts | 22 ++++++----------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/batch-transform.ts b/src/batch-transform.ts index 2be3165..9658220 100644 --- a/src/batch-transform.ts +++ b/src/batch-transform.ts @@ -80,12 +80,39 @@ export class BatchTransform extends TransformStream { }, }); - // Cap at maximum allowed values - this.maxBatchRecords = Math.min(args?.maxBatchRecords ?? 1000, 1000); - this.maxBatchBytes = Math.min( - args?.maxBatchBytes ?? 1024 * 1024, - 1024 * 1024, - ); + // Validate configuration + if (args?.maxBatchRecords !== undefined) { + if (args.maxBatchRecords < 1 || args.maxBatchRecords > 1000) { + throw new S2Error({ + message: `maxBatchRecords must be between 1 and 1000 (inclusive); got ${args.maxBatchRecords}`, + status: 400, + origin: "sdk", + }); + } + } + if (args?.maxBatchBytes !== undefined) { + const max = 1024 * 1024; + if (args.maxBatchBytes < 1 || args.maxBatchBytes > max) { + throw new S2Error({ + message: `maxBatchBytes must be between 1 and ${max} (1 MiB) bytes (inclusive); got ${args.maxBatchBytes}`, + status: 400, + origin: "sdk", + }); + } + } + if (args?.lingerDurationMillis !== undefined) { + if (args.lingerDurationMillis < 0) { + throw new S2Error({ + message: `lingerDurationMillis must be >= 0; got ${args.lingerDurationMillis}`, + status: 400, + origin: "sdk", + }); + } + } + + // Apply defaults + this.maxBatchRecords = args?.maxBatchRecords ?? 1000; + this.maxBatchBytes = args?.maxBatchBytes ?? 1024 * 1024; this.lingerDuration = args?.lingerDurationMillis ?? 5; this.fencing_token = args?.fencing_token; this.next_match_seq_num = args?.match_seq_num; diff --git a/src/tests/batch-transform.test.ts b/src/tests/batch-transform.test.ts index 55f1370..93ed47b 100644 --- a/src/tests/batch-transform.test.ts +++ b/src/tests/batch-transform.test.ts @@ -204,21 +204,13 @@ describe("BatchTransform", () => { reader.releaseLock(); }); - it("respects maximum limits (capped at 1000 records, 1 MiB)", () => { - // Should cap maxBatchRecords at 1000 - const batcher1 = new BatchTransform({ - maxBatchRecords: 5000, // Will be capped to 1000 - }); - // We can't directly access private fields, but we can verify behavior - - // Should cap maxBatchBytes at 1 MiB - const batcher2 = new BatchTransform({ - maxBatchBytes: 10 * 1024 * 1024, // Will be capped to 1 MiB - }); - - // Both should construct without error - expect(batcher1).toBeDefined(); - expect(batcher2).toBeDefined(); + it("rejects invalid configuration (records > 1000 or bytes > 1 MiB)", () => { + // maxBatchRecords > 1000 should throw + expect(() => new BatchTransform({ maxBatchRecords: 5000 })).toThrow(); + // maxBatchBytes > 1 MiB should throw + expect( + () => new BatchTransform({ maxBatchBytes: 10 * 1024 * 1024 }), + ).toThrow(); }); it("handles empty batches gracefully", async () => { From cebec28a7beb7f5c5626f51271357e53b6849b2f Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Wed, 12 Nov 2025 12:18:41 -0800 Subject: [PATCH 22/26] nits --- src/lib/stream/transport/fetch/index.ts | 2 +- src/lib/stream/transport/s2s/index.ts | 29 +++++++++---------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/lib/stream/transport/fetch/index.ts b/src/lib/stream/transport/fetch/index.ts index 82dc015..1353928 100644 --- a/src/lib/stream/transport/fetch/index.ts +++ b/src/lib/stream/transport/fetch/index.ts @@ -236,7 +236,7 @@ export class FetchReadSession // This ensures we don't wait forever if server stops sending events const result = await Promise.race([ reader.read(), - new Promise<{ done: true; value: undefined }>((_, reject) => + new Promise((_, reject) => setTimeout(() => { const elapsed = performance.now() - lastPingTimeMs; reject( diff --git a/src/lib/stream/transport/s2s/index.ts b/src/lib/stream/transport/s2s/index.ts index 8a50611..2df9401 100644 --- a/src/lib/stream/transport/s2s/index.ts +++ b/src/lib/stream/transport/s2s/index.ts @@ -720,29 +720,20 @@ class S2SAppendSession implements TransportAppendSession { const status = frame.statusCode ?? 500; try { const errorJson = JSON.parse(errorText); - if (status === 412) { - queueMicrotask(() => - safeError(makeAppendPreconditionError(status, errorJson)), - ); - } else { - queueMicrotask(() => - safeError( - makeServerError( + const err = + status === 412 + ? makeAppendPreconditionError(status, errorJson) + : makeServerError( { status, statusText: undefined }, errorJson, - ), - ), - ); - } + ); + queueMicrotask(() => safeError(err)); } catch { - queueMicrotask(() => - safeError( - makeServerError( - { status, statusText: undefined }, - errorText, - ), - ), + const err = makeServerError( + { status, statusText: undefined }, + errorText, ); + queueMicrotask(() => safeError(err)); } } stream.close(); From ea0806298037ee68f0f4014869f6766849d8809f Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Wed, 12 Nov 2025 12:21:33 -0800 Subject: [PATCH 23/26] can be private --- src/lib/stream/transport/s2s/framing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/stream/transport/s2s/framing.ts b/src/lib/stream/transport/s2s/framing.ts index 15c5e25..9384c03 100644 --- a/src/lib/stream/transport/s2s/framing.ts +++ b/src/lib/stream/transport/s2s/framing.ts @@ -80,7 +80,7 @@ export function frameMessage(opts: { * Parser for reading s2s frames from a stream */ export class S2SFrameParser { - buffer: Uint8Array = new Uint8Array(0); + private buffer: Uint8Array = new Uint8Array(0); /** * Add data to the parser buffer From eb04e1e53272c92d6e9a5b1c1f272bbed8b491ff Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Wed, 12 Nov 2025 23:03:46 -0800 Subject: [PATCH 24/26] session types --- src/lib/stream/types.ts | 28 +++++----------------------- src/tests/retryAppendSession.test.ts | 3 +-- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/lib/stream/types.ts b/src/lib/stream/types.ts index 1a735a8..3c70617 100644 --- a/src/lib/stream/types.ts +++ b/src/lib/stream/types.ts @@ -79,22 +79,8 @@ export interface TransportAppendSession { * Public AppendSession interface with retry, backpressure, and streams. * This is what users interact with - implemented by AppendSession. */ -export interface AppendSession - extends ReadableWritablePair, - AsyncDisposable { - submit( - records: AppendRecord | AppendRecord[], - args?: Omit & { precalculatedSize?: number }, - ): Promise; - acks(): AcksStream; - close(): Promise; - lastAckedPosition(): AppendAck | undefined; - /** - * If the session has failed, returns the original fatal error that caused - * the pump to stop. Returns undefined when the session has not failed. - */ - failureCause(): S2Error | undefined; -} +// Public AppendSession type is the concrete class from retry.ts +export type AppendSession = import("../retry.js").AppendSession; /** * Result type for transport-level read operations. @@ -122,13 +108,9 @@ export interface TransportReadSession< * Public-facing read session interface. * Yields records directly and propagates errors by throwing (standard stream behavior). */ -export interface ReadSession - extends ReadableStream>, - AsyncIterable>, - AsyncDisposable { - nextReadPosition(): StreamPosition | undefined; - lastObservedTail(): StreamPosition | undefined; -} +// Public ReadSession type is the concrete class from retry.ts +export type ReadSession = + import("../retry.js").ReadSession; export interface AppendSessionOptions { /** diff --git a/src/tests/retryAppendSession.test.ts b/src/tests/retryAppendSession.test.ts index 6528300..f06c523 100644 --- a/src/tests/retryAppendSession.test.ts +++ b/src/tests/retryAppendSession.test.ts @@ -8,14 +8,13 @@ import type { AcksStream, AppendArgs, AppendRecord, - AppendSession, TransportAppendSession, } from "../lib/stream/types.js"; /** * Minimal controllable AppendSession for testing AppendSessionImpl. */ -class FakeAppendSession implements AppendSession { +class FakeAppendSession { public readonly readable: ReadableStream; public readonly writable: WritableStream; private acksController!: ReadableStreamDefaultController; From ee0a110e37e4723be62748b51dcb3c3616155765 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Wed, 12 Nov 2025 23:22:03 -0800 Subject: [PATCH 25/26] a --- examples/image.ts | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/examples/image.ts b/examples/image.ts index 2d11912..ff37474 100644 --- a/examples/image.ts +++ b/examples/image.ts @@ -56,6 +56,20 @@ let image = await fetch( "https://upload.wikimedia.org/wikipedia/commons/2/24/Peter_Paul_Rubens_-_Self-portrait_-_RH.S.180_-_Rubenshuis_%28after_restoration%29.jpg", ); +function mapWithIndexAsync( + fn: (value: T, index: number) => Promise | U, +): TransformStream { + let index = 0; + + return new TransformStream({ + async transform(chunk, controller) { + const out = await fn(chunk, index); + index += 1; + controller.enqueue(out); + }, + }); +} + // Write directly from fetch response to S2 stream let append = await image .body! // Ensure each chunk is at most 128KiB. S2 has a maximum individual record size of 1MiB. @@ -68,10 +82,24 @@ let append = await image }, }), ) + .pipeThrough( + mapWithIndexAsync( + (record, index) => + ({ + ...record, + headers: [ + [ + new TextEncoder().encode("index"), + new TextEncoder().encode(index.toString()), + ], + ], + }) as AppendRecord, + ), + ) // Collect records into batches. .pipeThrough( new BatchTransform({ - lingerDurationMillis: 50, + lingerDurationMillis: 5, match_seq_num: startAt.tail.seq_num, }), ) From 0f3c4facf224b0bc60595fa834ea4ef2c1cc4b49 Mon Sep 17 00:00:00 2001 From: Stephen Balogh Date: Wed, 12 Nov 2025 23:38:08 -0800 Subject: [PATCH 26/26] a --- examples/throughput.ts | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 examples/throughput.ts diff --git a/examples/throughput.ts b/examples/throughput.ts new file mode 100644 index 0000000..180c87b --- /dev/null +++ b/examples/throughput.ts @@ -0,0 +1,69 @@ +import { createWriteStream } from "node:fs"; +import { + AppendRecord, + BatchTransform, + type ReadRecord, + S2, +} from "../src/index.js"; + +function createStringStream( + n: number, + delayMs: number = 0, +): ReadableStream { + let count = 0; + return new ReadableStream({ + async pull(controller) { + if (count < n) { + if (delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + console.log("pull", count); + const randomChars = Array.from({ length: 1024 * 10 }, () => + String.fromCharCode(97 + Math.floor(Math.random() * 26)), + ).join(""); + + var str = `${count} ${randomChars}`; + controller.enqueue(str); + count++; + } else { + controller.close(); + } + }, + }); +} + +const s2 = new S2({ + accessToken: process.env.S2_ACCESS_TOKEN!, + retry: { + maxAttempts: 10, + retryBackoffDurationMs: 100, + appendRetryPolicy: "noSideEffects", + requestTimeoutMillis: 10000, + }, +}); + +const basinName = process.env.S2_BASIN; +if (!basinName) { + console.error("S2_BASIN environment variable is not set"); + process.exit(1); +} + +const basin = s2.basin(basinName!); +const stream = basin.stream("throughput"); + +const sesh = await stream.appendSession({ maxQueuedBytes: 1024 * 1024 * 5 }); + +createStringStream(1000000, 0) + .pipeThrough( + new TransformStream({ + transform(arr, controller) { + controller.enqueue(AppendRecord.make(arr)); + }, + }), + ) + .pipeThrough( + new BatchTransform({ + lingerDurationMillis: 100, + }), + ) + .pipeTo(sesh.writable);