From d3d9ca68ab77eed4bcd16ee1c91ea25f1de2fbfe Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:26:18 -0500 Subject: [PATCH 1/7] ``` feat: add segment batch size and maximum concurrent segment batches options ``` --- lib/src/opentdf.ts | 10 + lib/tdf3/src/client/builders.ts | 12 + lib/tdf3/src/client/index.ts | 4 + lib/tdf3/src/tdf.ts | 341 ++++++++++++++++++++------ lib/tests/mocha/unit/builders.spec.ts | 15 +- lib/tests/mocha/unit/tdf.spec.ts | 93 +++++++ lib/tests/web/roundtrip.test.ts | 32 +++ 7 files changed, 433 insertions(+), 74 deletions(-) diff --git a/lib/src/opentdf.ts b/lib/src/opentdf.ts index 8c6009219..aa5b864a3 100644 --- a/lib/src/opentdf.ts +++ b/lib/src/opentdf.ts @@ -153,6 +153,12 @@ export type ReadOptions = { /** If set, prevents more than this number of concurrent requests to the KAS. */ concurrencyLimit?: number; + /** Maximum number of payload segments to fetch and decrypt per batch. */ + segmentBatchSize?: number; + + /** Maximum number of segment batches that may be fetched concurrently. */ + maxConcurrentSegmentBatches?: number; + /** Type of key to use for wrapping responses. */ wrappingKeyAlgorithm?: KasPublicKeyAlgorithm; }; @@ -495,7 +501,9 @@ class ZTDFReader { async decrypt(): Promise { const { assertionVerificationKeys, + maxConcurrentSegmentBatches, noVerify: noVerifyAssertions, + segmentBatchSize, wrappingKeyAlgorithm, } = this.opts; @@ -534,7 +542,9 @@ class ZTDFReader { keyMiddleware: async (k) => k, progressHandler: this.client.clientConfig.progressHandler, assertionVerificationKeys, + maxConcurrentSegmentBatches, noVerifyAssertions, + segmentBatchSize, wrappingKeyAlgorithm, fulfillableObligations: this.opts.fulfillableObligationFQNs || [], }, diff --git a/lib/tdf3/src/client/builders.ts b/lib/tdf3/src/client/builders.ts index 52800a692..96be55eda 100644 --- a/lib/tdf3/src/client/builders.ts +++ b/lib/tdf3/src/client/builders.ts @@ -536,6 +536,8 @@ export type DecryptParams = { streamMiddleware?: DecryptStreamMiddleware; assertionVerificationKeys?: AssertionVerificationKeys; concurrencyLimit?: number; + segmentBatchSize?: number; + maxConcurrentSegmentBatches?: number; noVerifyAssertions?: boolean; wrappingKeyAlgorithm?: KasPublicKeyAlgorithm; fulfillableObligationFQNs?: string[]; @@ -725,6 +727,16 @@ class DecryptParamsBuilder { return this; } + withSegmentBatchSize(size: number): DecryptParamsBuilder { + this._params.segmentBatchSize = size; + return this; + } + + withMaxConcurrentSegmentBatches(limit: number): DecryptParamsBuilder { + this._params.maxConcurrentSegmentBatches = limit; + return this; + } + /** * Generate a parameters object in the form expected by {@link Client#decrypt|decrypt}. *

diff --git a/lib/tdf3/src/client/index.ts b/lib/tdf3/src/client/index.ts index cbd51665b..f98c7e720 100644 --- a/lib/tdf3/src/client/index.ts +++ b/lib/tdf3/src/client/index.ts @@ -800,6 +800,8 @@ export class Client { assertionVerificationKeys, noVerifyAssertions, concurrencyLimit = 1, + maxConcurrentSegmentBatches, + segmentBatchSize, wrappingKeyAlgorithm, fulfillableObligationFQNs = [], }: DecryptParams): Promise { @@ -836,7 +838,9 @@ export class Client { keyMiddleware, progressHandler: this.clientConfig.progressHandler, assertionVerificationKeys, + maxConcurrentSegmentBatches, noVerifyAssertions, + segmentBatchSize, wrappingKeyAlgorithm, fulfillableObligations: fulfillableObligationFQNs, }) diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 6f615cd8f..12bbfb234 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -71,6 +71,10 @@ import { const DEFAULT_SEGMENT_SIZE = 1024 * 1024; const HEX_SEMVER_VERSION = '4.2.2'; +const LEGACY_SEGMENTS_PER_DOWNLOAD = 500; +const LEGACY_MAX_CONCURRENT_SEGMENT_BATCHES = 3; +const DEFAULT_BOUND_SEGMENT_BATCH_SIZE = 8; +const DEFAULT_BOUND_MAX_CONCURRENT_SEGMENT_BATCHES = 1; /** * Configuration for TDF3 @@ -181,6 +185,8 @@ export type DecryptConfiguration = { assertionVerificationKeys?: AssertionVerificationKeys; noVerifyAssertions?: boolean; concurrencyLimit?: number; + segmentBatchSize?: number; + maxConcurrentSegmentBatches?: number; wrappingKeyAlgorithm?: KasPublicKeyAlgorithm; }; @@ -1005,7 +1011,7 @@ async function decryptChunk( } async function updateChunkQueue( - chunkMap: Chunk[], + chunks: Chunk[], centralDirectory: CentralDirectory[], zipReader: ZipReader, reconstructedKey: SymmetricKey, @@ -1014,51 +1020,222 @@ async function updateChunkQueue( cryptoService: CryptoService, specVersion: string ) { - const chunksInOneDownload = 500; let requests = []; - const maxLength = 3; - for (let i = 0; i < chunkMap.length; i += chunksInOneDownload) { - if (requests.length === maxLength) { + for (let i = 0; i < chunks.length; i += LEGACY_SEGMENTS_PER_DOWNLOAD) { + if (requests.length === LEGACY_MAX_CONCURRENT_SEGMENT_BATCHES) { await Promise.all(requests); requests = []; } requests.push( - (async () => { - let buffer: Uint8Array | null; + fetchAndDecryptChunkSlice({ + centralDirectory, + zipReader, + reconstructedKey, + cipher, + segmentIntegrityAlgorithm, + cryptoService, + specVersion, + slice: chunks.slice(i, i + LEGACY_SEGMENTS_PER_DOWNLOAD), + }) + ); + } +} - const slice = chunkMap.slice(i, i + chunksInOneDownload); - try { - const bufferSize = slice.reduce( - (currentVal, { encryptedSegmentSize }) => currentVal + (encryptedSegmentSize as number), - 0 - ); - buffer = await zipReader.getPayloadSegment( - centralDirectory, - '0.payload', - slice[0].encryptedOffset, - bufferSize - ); - } catch (e) { - if (e instanceof InvalidFileError) { - throw e; - } - throw new NetworkError('unable to fetch payload segment', e); +function rejectChunks(chunks: Chunk[], error: Error) { + for (const chunk of chunks) { + chunk.decryptedChunk.reject(error); + } +} + +function asDecryptError(error: unknown, fallbackMessage: string): Error { + if (error instanceof Error) { + return error; + } + return new DecryptError(fallbackMessage, error); +} + +async function fetchAndDecryptChunkSlice({ + centralDirectory, + zipReader, + reconstructedKey, + cipher, + segmentIntegrityAlgorithm, + cryptoService, + specVersion, + slice, +}: { + centralDirectory: CentralDirectory[]; + zipReader: ZipReader; + reconstructedKey: SymmetricKey; + cipher: SymmetricCipher; + segmentIntegrityAlgorithm: IntegrityAlgorithm; + cryptoService: CryptoService; + specVersion: string; + slice: Chunk[]; +}) { + let buffer!: Uint8Array; + try { + const bufferSize = slice.reduce( + (currentVal, { encryptedSegmentSize }) => currentVal + (encryptedSegmentSize as number), + 0 + ); + buffer = await zipReader.getPayloadSegment( + centralDirectory, + '0.payload', + slice[0].encryptedOffset, + bufferSize + ); + } catch (error) { + const wrappedError = + error instanceof InvalidFileError + ? error + : new NetworkError('unable to fetch payload segment', error); + rejectChunks(slice, wrappedError); + return; + } + + try { + await sliceAndDecrypt({ + buffer, + cryptoService, + reconstructedKey, + slice, + cipher, + segmentIntegrityAlgorithm, + specVersion, + }); + } catch (error) { + rejectChunks(slice, asDecryptError(error, 'failed to decrypt payload segment')); + } +} + +export type SegmentBatchSchedulerState = { + consumedSegments: number; + inFlightBatches: number; + maxPrefetchedSegments: number; + scheduledSegments: number; +}; + +export type SegmentBatchScheduler = { + fillWindow: () => void; + markConsumed: (count?: number) => void; + snapshot: () => SegmentBatchSchedulerState; +}; + +export function createBoundedSegmentScheduler({ + totalSegments, + segmentBatchSize, + maxConcurrentSegmentBatches, + onError, + scheduleBatch, +}: { + totalSegments: number; + segmentBatchSize: number; + maxConcurrentSegmentBatches: number; + onError?: (error: Error, startIndex: number, endIndex: number) => void; + scheduleBatch: (startIndex: number, endIndex: number) => Promise; +}): SegmentBatchScheduler { + const maxPrefetchedSegments = segmentBatchSize * maxConcurrentSegmentBatches; + let consumedSegments = 0; + let inFlightBatches = 0; + let pumping = false; + let scheduledSegments = 0; + let stopped = false; + + const pump = () => { + if (pumping || stopped) { + return; + } + pumping = true; + try { + while ( + !stopped && + inFlightBatches < maxConcurrentSegmentBatches && + scheduledSegments < totalSegments + ) { + const prefetchedSegments = scheduledSegments - consumedSegments; + const remainingWindow = maxPrefetchedSegments - prefetchedSegments; + if (remainingWindow <= 0) { + break; } - if (buffer) { - sliceAndDecrypt({ - buffer, - cryptoService, - reconstructedKey, - slice, - cipher, - segmentIntegrityAlgorithm, - specVersion, - }); + + const startIndex = scheduledSegments; + const nextBatchSize = Math.min(segmentBatchSize, totalSegments - startIndex); + if (remainingWindow < nextBatchSize) { + break; } - })() - ); + const endIndex = startIndex + nextBatchSize; + scheduledSegments = endIndex; + inFlightBatches += 1; + + void Promise.resolve() + .then(() => scheduleBatch(startIndex, endIndex)) + .catch((error) => { + stopped = true; + onError?.(asDecryptError(error, 'failed to schedule segment batch'), startIndex, endIndex); + }) + .finally(() => { + inFlightBatches -= 1; + pump(); + }); + } + } finally { + pumping = false; + } + }; + + return { + fillWindow() { + pump(); + }, + markConsumed(count = 1) { + consumedSegments = Math.min(totalSegments, consumedSegments + count); + pump(); + }, + snapshot() { + return { + consumedSegments, + inFlightBatches, + maxPrefetchedSegments, + scheduledSegments, + }; + }, + }; +} + +function normalizeSegmentBatchSetting( + value: number | undefined, + defaultValue: number, + name: 'segmentBatchSize' | 'maxConcurrentSegmentBatches' +) { + const normalized = value ?? defaultValue; + if (!Number.isInteger(normalized) || normalized < 1) { + throw new ConfigurationError(`${name} must be a positive integer`); } + return normalized; +} + +function getBoundedSegmentSchedulerOptions({ + segmentBatchSize, + maxConcurrentSegmentBatches, +}: Pick) { + if (segmentBatchSize === undefined && maxConcurrentSegmentBatches === undefined) { + return undefined; + } + + return { + segmentBatchSize: normalizeSegmentBatchSetting( + segmentBatchSize, + DEFAULT_BOUND_SEGMENT_BATCH_SIZE, + 'segmentBatchSize' + ), + maxConcurrentSegmentBatches: normalizeSegmentBatchSetting( + maxConcurrentSegmentBatches, + DEFAULT_BOUND_MAX_CONCURRENT_SEGMENT_BATCHES, + 'maxConcurrentSegmentBatches' + ), + }; } export async function sliceAndDecrypt({ @@ -1083,9 +1260,7 @@ export async function sliceAndDecrypt({ const offset = slice[0].encryptedOffset === 0 ? encryptedOffset : encryptedOffset % slice[0].encryptedOffset; - const encryptedChunk = new Uint8Array( - buffer.slice(offset, offset + (encryptedSegmentSize as number)) - ); + const encryptedChunk = buffer.subarray(offset, offset + (encryptedSegmentSize as number)); if (encryptedChunk.length !== encryptedSegmentSize) { throw new DecryptError('Failed to fetch entire segment'); @@ -1212,27 +1387,22 @@ export async function decryptStreamFrom( } let mapOfRequestsOffset = 0; - const chunkMap = new Map( - segments.map( - ({ + const chunks = segments.map( + ({ + hash, + encryptedSegmentSize = encryptedSegmentSizeDefault, + segmentSize = segmentSizeDefault, + }) => { + const chunk: Chunk = { hash, - encryptedSegmentSize = encryptedSegmentSizeDefault, - segmentSize = segmentSizeDefault, - }) => { - const result = (() => { - const chunk: Chunk = { - hash, - encryptedOffset: mapOfRequestsOffset, - encryptedSegmentSize, - decryptedChunk: mailbox(), - plainSegmentSize: segmentSize, - }; - return chunk; - })(); - mapOfRequestsOffset += encryptedSegmentSize; - return [hash, result]; - } - ) + encryptedOffset: mapOfRequestsOffset, + encryptedSegmentSize, + decryptedChunk: mailbox(), + plainSegmentSize: segmentSize, + }; + mapOfRequestsOffset += encryptedSegmentSize; + return chunk; + } ); const cipher = new AesGcmCipher(cfg.cryptoService); @@ -1241,33 +1411,58 @@ export async function decryptStreamFrom( throw new UnsupportedError(`Unsupported segment hash alg [${segmentIntegrityAlg}]`); } - // Not waiting for Promise to resolve - updateChunkQueue( - Array.from(chunkMap.values()), - centralDirectory, - zipReader, - keyForDecryption, - cipher, - segmentIntegrityAlg, - cfg.cryptoService, - specVersion - ); + const schedulerOptions = getBoundedSegmentSchedulerOptions(cfg); + let scheduler: SegmentBatchScheduler | undefined; + if (schedulerOptions) { + scheduler = createBoundedSegmentScheduler({ + totalSegments: chunks.length, + ...schedulerOptions, + onError: (error, startIndex) => { + rejectChunks(chunks.slice(startIndex), error); + }, + scheduleBatch: async (startIndex, endIndex) => + fetchAndDecryptChunkSlice({ + centralDirectory, + zipReader, + reconstructedKey: keyForDecryption, + cipher, + segmentIntegrityAlgorithm: segmentIntegrityAlg, + cryptoService: cfg.cryptoService, + specVersion, + slice: chunks.slice(startIndex, endIndex), + }), + }); + scheduler.fillWindow(); + } else { + void updateChunkQueue( + chunks, + centralDirectory, + zipReader, + keyForDecryption, + cipher, + segmentIntegrityAlg, + cfg.cryptoService, + specVersion + ); + } let progress = 0; + let nextChunkIndex = 0; const underlyingSource = { pull: async (controller: ReadableStreamDefaultController) => { - if (chunkMap.size === 0) { + if (nextChunkIndex >= chunks.length) { controller.close(); return; } - const [hash, chunk] = chunkMap.entries().next().value; + const chunk = chunks[nextChunkIndex]; const decryptedSegment = await chunk.decryptedChunk; controller.enqueue(new Uint8Array(decryptedSegment.payload.asByteArray())); progress += chunk.encryptedSegmentSize; cfg.progressHandler?.(progress); - chunkMap.delete(hash); + nextChunkIndex += 1; + scheduler?.markConsumed(); }, ...(cfg.fileStreamServiceWorker && { fileStreamServiceWorker: cfg.fileStreamServiceWorker }), }; diff --git a/lib/tests/mocha/unit/builders.spec.ts b/lib/tests/mocha/unit/builders.spec.ts index 0079fdc7e..de177731a 100644 --- a/lib/tests/mocha/unit/builders.spec.ts +++ b/lib/tests/mocha/unit/builders.spec.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { type AttributeObject } from '../../../tdf3/src/models/attribute.js'; -import { EncryptParamsBuilder } from '../../../tdf3/src/client/builders.js'; +import { DecryptParamsBuilder, EncryptParamsBuilder } from '../../../tdf3/src/client/builders.js'; const aex = { kasUrl: 'https://kas', @@ -56,3 +56,16 @@ describe('EncryptParamsBuilder', () => { }); }); }); + +describe('DecryptParamsBuilder', () => { + it('should store segment scheduler tuning', () => { + const params = new DecryptParamsBuilder() + .withStringSource('ciphertext') + .withSegmentBatchSize(4) + .withMaxConcurrentSegmentBatches(2) + .build(); + + expect(params.segmentBatchSize).to.equal(4); + expect(params.maxConcurrentSegmentBatches).to.equal(2); + }); +}); diff --git a/lib/tests/mocha/unit/tdf.spec.ts b/lib/tests/mocha/unit/tdf.spec.ts index 55cf84968..556f32b70 100644 --- a/lib/tests/mocha/unit/tdf.spec.ts +++ b/lib/tests/mocha/unit/tdf.spec.ts @@ -152,6 +152,99 @@ describe('validatePolicyObject', () => { }); }); +function createDeferred() { + let reject!: (error: unknown) => void; + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((innerResolve, innerReject) => { + resolve = innerResolve; + reject = innerReject; + }); + return { promise, reject, resolve }; +} + +async function flushScheduler() { + await Promise.resolve(); + await Promise.resolve(); +} + +describe('bounded segment scheduler', () => { + it('honors the configured batch size', async () => { + const started: Array<[number, number]> = []; + const deferred = createDeferred(); + const scheduler = TDF.createBoundedSegmentScheduler({ + totalSegments: 6, + segmentBatchSize: 2, + maxConcurrentSegmentBatches: 1, + scheduleBatch: (startIndex, endIndex) => { + started.push([startIndex, endIndex]); + return deferred.promise; + }, + }); + + scheduler.fillWindow(); + await flushScheduler(); + expect(started).to.deep.equal([[0, 2]]); + + deferred.resolve(); + await flushScheduler(); + + scheduler.markConsumed(); + await flushScheduler(); + expect(started).to.deep.equal([[0, 2]]); + + scheduler.markConsumed(); + await flushScheduler(); + expect(started).to.deep.equal([ + [0, 2], + [2, 4], + ]); + }); + + it('bounds scheduled work to the configured prefetch window', async () => { + const started: Array<[number, number]> = []; + const deferredBatches = [createDeferred(), createDeferred(), createDeferred()]; + let nextDeferred = 0; + const scheduler = TDF.createBoundedSegmentScheduler({ + totalSegments: 10, + segmentBatchSize: 2, + maxConcurrentSegmentBatches: 2, + scheduleBatch: (startIndex, endIndex) => { + started.push([startIndex, endIndex]); + return deferredBatches[nextDeferred++].promise; + }, + }); + + scheduler.fillWindow(); + await flushScheduler(); + + expect(started).to.deep.equal([ + [0, 2], + [2, 4], + ]); + expect(scheduler.snapshot()).to.deep.equal({ + consumedSegments: 0, + inFlightBatches: 2, + maxPrefetchedSegments: 4, + scheduledSegments: 4, + }); + + deferredBatches[0].resolve(); + await flushScheduler(); + expect(started).to.deep.equal([ + [0, 2], + [2, 4], + ]); + + scheduler.markConsumed(2); + await flushScheduler(); + expect(started).to.deep.equal([ + [0, 2], + [2, 4], + [4, 6], + ]); + }); +}); + describe('splitLookupTableFactory', () => { it('should return a correct split table for valid input', () => { const keyAccess: KeyAccessObject[] = [ diff --git a/lib/tests/web/roundtrip.test.ts b/lib/tests/web/roundtrip.test.ts index d9ff9b482..3ab402a7a 100644 --- a/lib/tests/web/roundtrip.test.ts +++ b/lib/tests/web/roundtrip.test.ts @@ -145,4 +145,36 @@ describe('Local roundtrip Tests', () => { const decoded = new TextDecoder().decode(actual); expect(decoded).to.be.equal('hello world'); }); + + it(`ztdf roundtrip chunker decrypt with bounded segment scheduler`, async () => { + const client = new OpenTDF({ + authProvider, + defaultReadOptions: { + allowedKASEndpoints: [kasEndpoint], + }, + }); + const plainText = 'chunker-download-'.repeat(512); + const cipherTextStream = await client.createZTDF({ + autoconfigure: false, + defaultKASEndpoint: kasEndpoint, + source: { type: 'buffer', location: new TextEncoder().encode(plainText) }, + windowSize: 1024, + }); + const cipherTextArray = new Uint8Array(await new Response(cipherTextStream).arrayBuffer()); + const trackingChunker = async (byteStart?: number, byteEnd?: number) => { + if (byteStart === undefined) { + return cipherTextArray.slice(); + } + return cipherTextArray.slice(byteStart, byteEnd); + }; + + const ztdfParsed = await client.read({ + source: { type: 'chunker', location: trackingChunker }, + segmentBatchSize: 2, + maxConcurrentSegmentBatches: 1, + }); + + const actual = await new Response(ztdfParsed).arrayBuffer(); + expect(new TextDecoder().decode(actual)).to.be.equal(plainText); + }); }); From 9cb27307c9a6b6b70cd0f072c798d3c92e3182dc Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:40:07 -0500 Subject: [PATCH 2/7] lint and audit fixes --- lib/package-lock.json | 167 +++++++++++++++++++++--------------------- lib/tdf3/src/tdf.ts | 5 +- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/lib/package-lock.json b/lib/package-lock.json index 6e787a42f..f1634826d 100644 --- a/lib/package-lock.json +++ b/lib/package-lock.json @@ -69,7 +69,6 @@ "version": "1.2.6", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -134,6 +133,7 @@ "version": "7.26.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.0", @@ -515,7 +515,8 @@ "node_modules/@bufbuild/protobuf": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.3.0.tgz", - "integrity": "sha512-WK6zH4MtBp/uesX8KGCnwDDRVnEVHUvwjsigKXcSR57Oo8Oyv1vRS9qyUlSP+6KWRl5z8tNAU5qpf3QodeVYxA==" + "integrity": "sha512-WK6zH4MtBp/uesX8KGCnwDDRVnEVHUvwjsigKXcSR57Oo8Oyv1vRS9qyUlSP+6KWRl5z8tNAU5qpf3QodeVYxA==", + "peer": true }, "node_modules/@bufbuild/protoc-gen-es": { "version": "2.2.5", @@ -583,6 +584,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.0.2.tgz", "integrity": "sha512-xZuylIUNvNlH52e/4eQsZvY4QZyDJRtEFEDnn/yBrv5Xi5ZZI/p8X+GAHH35ucVaBvv9u7OzHZo8+tEh1EFTxA==", + "peer": true, "peerDependencies": { "@bufbuild/protobuf": "^2.2.0" } @@ -684,6 +686,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -707,6 +710,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -1177,7 +1181,6 @@ "version": "2.1.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", @@ -1239,7 +1242,6 @@ "version": "0.13.0", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", @@ -1253,7 +1255,6 @@ "version": "1.0.1", "dev": true, "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=12.22" }, @@ -1265,8 +1266,7 @@ "node_modules/@humanwhocodes/object-schema": { "version": "2.0.3", "dev": true, - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/@isaacs/cliui": { "version": "8.0.2", @@ -2352,6 +2352,7 @@ "version": "9.6.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "*", "@types/json-schema": "*" @@ -2714,6 +2715,7 @@ "integrity": "sha512-BtE0k6cjwjLZoZixN0t5AKP0kSzlGu7FctRXYuPAm//aaiZhmfq1JwdYpYr1brzEspYyFeF+8XF5j2VK6oalrA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.54.0", "@typescript-eslint/types": "8.54.0", @@ -2912,9 +2914,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -3025,8 +3027,7 @@ "node_modules/@ungap/structured-clone": { "version": "1.3.0", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/@web/browser-logs": { "version": "0.4.0", @@ -3568,6 +3569,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3592,7 +3594,6 @@ "version": "5.3.2", "dev": true, "license": "MIT", - "peer": true, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } @@ -3623,7 +3624,6 @@ "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -3989,9 +3989,9 @@ } }, "node_modules/basic-ftp": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", - "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz", + "integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==", "dev": true, "license": "MIT", "engines": { @@ -4045,9 +4045,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "dev": true, "license": "MIT", "dependencies": { @@ -4091,6 +4091,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -4164,9 +4165,9 @@ } }, "node_modules/c8/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -4334,7 +4335,6 @@ "version": "3.1.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -4928,8 +4928,7 @@ "node_modules/deep-is": { "version": "0.1.4", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/deepmerge": { "version": "4.3.1", @@ -5040,7 +5039,8 @@ "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1551306.tgz", "integrity": "sha512-CFx8QdSim8iIv+2ZcEOclBKTQY6BI1IEDa7Tm9YkwAXzEWFndTEzpTo5jAUhSnq24IC7xaDw0wvGcm96+Y3PEg==", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/di": { "version": "0.0.1", @@ -5072,7 +5072,6 @@ "version": "3.0.0", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "esutils": "^2.0.2" }, @@ -5473,6 +5472,7 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -5550,7 +5550,6 @@ "version": "8.57.1", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } @@ -5559,7 +5558,6 @@ "version": "7.2.2", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" @@ -5575,7 +5573,6 @@ "version": "5.3.0", "dev": true, "license": "BSD-2-Clause", - "peer": true, "engines": { "node": ">=4.0" } @@ -5584,7 +5581,6 @@ "version": "6.0.2", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "is-glob": "^4.0.3" }, @@ -5596,7 +5592,6 @@ "version": "9.6.1", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", @@ -5625,7 +5620,6 @@ "version": "1.5.0", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "estraverse": "^5.1.0" }, @@ -5637,7 +5631,6 @@ "version": "5.3.0", "dev": true, "license": "BSD-2-Clause", - "peer": true, "engines": { "node": ">=4.0" } @@ -5839,14 +5832,12 @@ "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/fast-uri": { "version": "3.1.0", @@ -5895,7 +5886,6 @@ "version": "6.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.0.4" }, @@ -6009,7 +5999,6 @@ "version": "3.2.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", @@ -6304,9 +6293,9 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { @@ -6336,7 +6325,6 @@ "version": "13.24.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "type-fest": "^0.20.2" }, @@ -6387,8 +6375,7 @@ "node_modules/graphemer": { "version": "1.4.0", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/has-flag": { "version": "4.0.0", @@ -6626,7 +6613,6 @@ "version": "3.3.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -7359,8 +7345,7 @@ "node_modules/json-buffer": { "version": "3.0.1", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-canonicalize": { "version": "1.0.6", @@ -7374,14 +7359,12 @@ "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json5": { "version": "2.2.3", @@ -7592,7 +7575,6 @@ "version": "4.5.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "json-buffer": "3.0.1" } @@ -7750,7 +7732,6 @@ "version": "0.4.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" @@ -7887,8 +7868,7 @@ "node_modules/lodash.merge": { "version": "4.6.2", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/log-symbols": { "version": "4.1.0", @@ -8183,9 +8163,9 @@ } }, "node_modules/mocha/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -8691,7 +8671,6 @@ "version": "0.9.3", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@aashutoshrathi/word-wrap": "^1.2.3", "deep-is": "^0.1.3", @@ -8843,7 +8822,6 @@ "version": "1.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "callsites": "^3.0.0" }, @@ -9069,7 +9047,6 @@ "version": "1.2.1", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8.0" } @@ -9080,6 +9057,7 @@ "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -9366,9 +9344,9 @@ } }, "node_modules/read-installed-packages/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -9635,7 +9613,6 @@ "version": "4.0.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=4" } @@ -9749,6 +9726,7 @@ "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -9878,6 +9856,7 @@ "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -9923,9 +9902,9 @@ } }, "node_modules/serialize-javascript": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.4.tgz", - "integrity": "sha512-DuGdB+Po43Q5Jxwpzt1lhyFSYKryqoNjQSA9M92tyw0lyHIOur+XCalOUe0KTJpyqzT8+fQ5A0Jf7vCx/NKmIg==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.5.tgz", + "integrity": "sha512-F4LcB0UqUl1zErq+1nYEEzSHJnIwb3AF2XWB94b+afhrekOUijwooAYqFyRbjYkm2PAKBabx6oYv/xDxNi8IBw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -10184,17 +10163,37 @@ } }, "node_modules/socket.io-parser": { - "version": "4.2.4", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.6.tgz", + "integrity": "sha512-asJqbVBDsBCJx0pTqw3WfesSY0iRX+2xzWEWzrpcH7L6fLzrhyF8WPI8UaeM4YCuDfpwA/cgsdugMsmtz8EJeg==", "dev": true, "license": "MIT", "dependencies": { "@socket.io/component-emitter": "~3.1.0", - "debug": "~4.3.1" + "debug": "~4.4.1" }, "engines": { "node": ">=10.0.0" } }, + "node_modules/socket.io-parser/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/socks": { "version": "2.8.7", "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", @@ -10703,8 +10702,7 @@ "node_modules/text-table": { "version": "0.2.0", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/through": { "version": "2.3.8", @@ -10762,6 +10760,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -10887,7 +10886,6 @@ "version": "0.4.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "prelude-ls": "^1.2.1" }, @@ -10909,7 +10907,6 @@ "version": "0.20.2", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -10982,9 +10979,9 @@ } }, "node_modules/typedoc/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -11013,6 +11010,7 @@ "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -11143,7 +11141,6 @@ "version": "4.4.1", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "punycode": "^2.1.0" } @@ -11260,6 +11257,7 @@ "integrity": "sha512-dRXm0a2qcHPUBEzVk8uph0xWSjV/xZxenQQbLwnwP7caQCYpqG1qddwlyEkIDkYn0K8tvmcrZ+bOrzoQ3HxCDw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -11309,6 +11307,7 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -11568,9 +11567,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "dev": true, "license": "ISC", "bin": { diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 12bbfb234..0744eb1be 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -1052,7 +1052,7 @@ function asDecryptError(error: unknown, fallbackMessage: string): Error { if (error instanceof Error) { return error; } - return new DecryptError(fallbackMessage, error); + return new DecryptError(fallbackMessage, new Error(String(error))); } async function fetchAndDecryptChunkSlice({ @@ -1457,9 +1457,10 @@ export async function decryptStreamFrom( const chunk = chunks[nextChunkIndex]; const decryptedSegment = await chunk.decryptedChunk; + const encryptedSegmentSize = chunk.encryptedSegmentSize ?? 0; controller.enqueue(new Uint8Array(decryptedSegment.payload.asByteArray())); - progress += chunk.encryptedSegmentSize; + progress += encryptedSegmentSize; cfg.progressHandler?.(progress); nextChunkIndex += 1; scheduler?.markConsumed(); From 3ecb6b847f19b1e1fb3d51950472834216abf58b Mon Sep 17 00:00:00 2001 From: bnrobinson93 <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:42:17 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=A4=96=20=F0=9F=8E=A8=20Autoformat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tdf3/src/tdf.ts | 6 +++++- lib/tests/mocha/unit/tdf.spec.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 0744eb1be..7c86858c5 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -1173,7 +1173,11 @@ export function createBoundedSegmentScheduler({ .then(() => scheduleBatch(startIndex, endIndex)) .catch((error) => { stopped = true; - onError?.(asDecryptError(error, 'failed to schedule segment batch'), startIndex, endIndex); + onError?.( + asDecryptError(error, 'failed to schedule segment batch'), + startIndex, + endIndex + ); }) .finally(() => { inFlightBatches -= 1; diff --git a/lib/tests/mocha/unit/tdf.spec.ts b/lib/tests/mocha/unit/tdf.spec.ts index 556f32b70..2e476ac3b 100644 --- a/lib/tests/mocha/unit/tdf.spec.ts +++ b/lib/tests/mocha/unit/tdf.spec.ts @@ -202,7 +202,11 @@ describe('bounded segment scheduler', () => { it('bounds scheduled work to the configured prefetch window', async () => { const started: Array<[number, number]> = []; - const deferredBatches = [createDeferred(), createDeferred(), createDeferred()]; + const deferredBatches = [ + createDeferred(), + createDeferred(), + createDeferred(), + ]; let nextDeferred = 0; const scheduler = TDF.createBoundedSegmentScheduler({ totalSegments: 10, From c678f88d2696294a7de9613db133bb120793efc5 Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:49:22 -0500 Subject: [PATCH 4/7] code rabbit fixes --- lib/src/opentdf.ts | 10 ++++++++-- lib/tdf3/src/client/builders.ts | 8 ++++++++ lib/tdf3/src/tdf.ts | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/src/opentdf.ts b/lib/src/opentdf.ts index aa5b864a3..738ab10dc 100644 --- a/lib/src/opentdf.ts +++ b/lib/src/opentdf.ts @@ -153,10 +153,16 @@ export type ReadOptions = { /** If set, prevents more than this number of concurrent requests to the KAS. */ concurrencyLimit?: number; - /** Maximum number of payload segments to fetch and decrypt per batch. */ + /** + * Maximum number of payload segments to fetch and decrypt per batch. + * Adjust together with `maxConcurrentSegmentBatches` for expected throughput. + */ segmentBatchSize?: number; - /** Maximum number of segment batches that may be fetched concurrently. */ + /** + * Maximum number of segment batches that may be fetched concurrently. + * Adjust together with `segmentBatchSize` for expected throughput. + */ maxConcurrentSegmentBatches?: number; /** Type of key to use for wrapping responses. */ diff --git a/lib/tdf3/src/client/builders.ts b/lib/tdf3/src/client/builders.ts index 96be55eda..863781ec1 100644 --- a/lib/tdf3/src/client/builders.ts +++ b/lib/tdf3/src/client/builders.ts @@ -727,11 +727,19 @@ class DecryptParamsBuilder { return this; } + /** + * Set the number of payload segments to fetch and decrypt per batch. + * Adjust together with `withMaxConcurrentSegmentBatches()` for expected throughput. + */ withSegmentBatchSize(size: number): DecryptParamsBuilder { this._params.segmentBatchSize = size; return this; } + /** + * Set the maximum number of payload segment batches to fetch concurrently. + * Adjust together with `withSegmentBatchSize()` for expected throughput. + */ withMaxConcurrentSegmentBatches(limit: number): DecryptParamsBuilder { this._params.maxConcurrentSegmentBatches = limit; return this; diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 7c86858c5..8d5a02742 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -73,8 +73,8 @@ const DEFAULT_SEGMENT_SIZE = 1024 * 1024; const HEX_SEMVER_VERSION = '4.2.2'; const LEGACY_SEGMENTS_PER_DOWNLOAD = 500; const LEGACY_MAX_CONCURRENT_SEGMENT_BATCHES = 3; -const DEFAULT_BOUND_SEGMENT_BATCH_SIZE = 8; -const DEFAULT_BOUND_MAX_CONCURRENT_SEGMENT_BATCHES = 1; +const DEFAULT_BOUND_SEGMENT_BATCH_SIZE = LEGACY_SEGMENTS_PER_DOWNLOAD; +const DEFAULT_BOUND_MAX_CONCURRENT_SEGMENT_BATCHES = LEGACY_MAX_CONCURRENT_SEGMENT_BATCHES; /** * Configuration for TDF3 @@ -1037,7 +1037,7 @@ async function updateChunkQueue( cryptoService, specVersion, slice: chunks.slice(i, i + LEGACY_SEGMENTS_PER_DOWNLOAD), - }) + }).catch(() => undefined) ); } } @@ -1092,7 +1092,7 @@ async function fetchAndDecryptChunkSlice({ ? error : new NetworkError('unable to fetch payload segment', error); rejectChunks(slice, wrappedError); - return; + throw wrappedError; } try { @@ -1106,7 +1106,9 @@ async function fetchAndDecryptChunkSlice({ specVersion, }); } catch (error) { - rejectChunks(slice, asDecryptError(error, 'failed to decrypt payload segment')); + const wrappedError = asDecryptError(error, 'failed to decrypt payload segment'); + rejectChunks(slice, wrappedError); + throw wrappedError; } } @@ -1220,6 +1222,12 @@ function normalizeSegmentBatchSetting( return normalized; } +/** + * Enables bounded scheduling only when at least one tuning knob is set. + * If callers set only one knob, the other falls back to the legacy value so + * throughput stays aligned with the pre-bounded path. Adjust both knobs together + * when tuning for predictable memory and performance characteristics. + */ function getBoundedSegmentSchedulerOptions({ segmentBatchSize, maxConcurrentSegmentBatches, From c4cf7ade1c8d564853d3895c022cc51c3cce1eb3 Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:29:33 -0500 Subject: [PATCH 5/7] tests: decrypt read tuning --- web-app/src/App.tsx | 36 ++++++++++++++++++++++++++++++++ web-app/tests/tests/huge.spec.ts | 16 ++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 171362ab5..d8af79ea3 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -82,6 +82,37 @@ type RandomInputSource = { type InputSource = FileInputSource | UrlInputSource | RandomInputSource; type SinkType = 'file' | 'fsapi' | 'none'; +type DecryptReadTuning = { + segmentBatchSize?: number; + maxConcurrentSegmentBatches?: number; +}; + +function readPositiveIntSearchParam(params: URLSearchParams, name: string): number | undefined { + const raw = params.get(name); + if (!raw) { + return undefined; + } + const parsed = Number.parseInt(raw, 10); + if (!Number.isInteger(parsed) || parsed < 1) { + console.warn(`Ignoring invalid ${name} query param: ${raw}`); + return undefined; + } + return parsed; +} + +function getDecryptReadTuningFromLocation(): DecryptReadTuning { + const params = new URLSearchParams(window.location.search); + const segmentBatchSize = readPositiveIntSearchParam(params, 'segmentBatchSize'); + const maxConcurrentSegmentBatches = readPositiveIntSearchParam( + params, + 'maxConcurrentSegmentBatches' + ); + + return { + ...(segmentBatchSize !== undefined && { segmentBatchSize }), + ...(maxConcurrentSegmentBatches !== undefined && { maxConcurrentSegmentBatches }), + }; +} function fileNameFor(inputSource: InputSource) { if (!inputSource) { @@ -409,10 +440,15 @@ function App() { if (sinkType === 'fsapi') { f = await getNewFileHandle(decryptedFileExtension(fileNameFor(inputSource)), dfn); } + const decryptReadTuning = getDecryptReadTuningFromLocation(); + if (Object.keys(decryptReadTuning).length > 0) { + console.info(`Using decrypt read tuning ${JSON.stringify(decryptReadTuning)}`); + } const client = new OpenTDF({ authProvider: oidcClient, defaultReadOptions: { allowedKASEndpoints: [config.kas], + ...decryptReadTuning, }, dpopKeys: oidcClient.getSigningKey(), }); diff --git a/web-app/tests/tests/huge.spec.ts b/web-app/tests/tests/huge.spec.ts index 764631079..49d09efd4 100644 --- a/web-app/tests/tests/huge.spec.ts +++ b/web-app/tests/tests/huge.spec.ts @@ -2,6 +2,8 @@ import { test, expect, type Page } from '@playwright/test'; import fs from 'node:fs'; import { authorize, loadFile } from './acts.js'; +const appUrl = 'http://localhost:65432/'; + test.beforeEach(async ({ page }) => { page.on('pageerror', (err) => { console.error(err); @@ -13,6 +15,17 @@ test.beforeEach(async ({ page }) => { test('Large File', async ({ page }) => { await authorize(page); + await page.goto(`${appUrl}?segmentBatchSize=2&maxConcurrentSegmentBatches=1`); + await expect(page.locator('#sessionState')).toHaveText('loggedin'); + + const decryptTuningLogs: string[] = []; + page.on('console', (message) => { + const text = message.text(); + if (text.includes('Using decrypt read tuning')) { + decryptTuningLogs.push(text); + } + }); + const threeGigs = 3 * 2 ** 30; await page.locator('#randomSelector').fill(threeGigs.toString()); @@ -36,6 +49,9 @@ test('Large File', async ({ page }) => { await page.locator('#decryptButton').click(); const download2 = await plainDownloadPromise; expect(download2.suggestedFilename()).toContain('.decrypted'); + expect(decryptTuningLogs).toEqual([ + 'Using decrypt read tuning {"segmentBatchSize":2,"maxConcurrentSegmentBatches":1}', + ]); const plainTextPath = await download2.path(); if (!plainTextPath) { throw new Error(); From abd50cc5e0bf063a9003b563d5bd88898fa2aa39 Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:32:18 -0500 Subject: [PATCH 6/7] remove hardcoded appUrl --- web-app/tests/tests/huge.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web-app/tests/tests/huge.spec.ts b/web-app/tests/tests/huge.spec.ts index 49d09efd4..e9822a6e1 100644 --- a/web-app/tests/tests/huge.spec.ts +++ b/web-app/tests/tests/huge.spec.ts @@ -2,8 +2,6 @@ import { test, expect, type Page } from '@playwright/test'; import fs from 'node:fs'; import { authorize, loadFile } from './acts.js'; -const appUrl = 'http://localhost:65432/'; - test.beforeEach(async ({ page }) => { page.on('pageerror', (err) => { console.error(err); From 8b480ab568255ed5bb0753f36e9a0fe6904d6b0b Mon Sep 17 00:00:00 2001 From: Brad R <31802085+bnrobinson93@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:52:15 -0500 Subject: [PATCH 7/7] code rabbit fix --- web-app/src/App.tsx | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index d8af79ea3..9d315e7b3 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -86,12 +86,18 @@ type DecryptReadTuning = { segmentBatchSize?: number; maxConcurrentSegmentBatches?: number; }; +const MAX_SEGMENT_BATCH_SIZE = 500; +const MAX_CONCURRENT_SEGMENT_BATCHES = 3; function readPositiveIntSearchParam(params: URLSearchParams, name: string): number | undefined { const raw = params.get(name); if (!raw) { return undefined; } + if (!/^\d+$/.test(raw)) { + console.warn(`Ignoring invalid ${name} query param: ${raw}`); + return undefined; + } const parsed = Number.parseInt(raw, 10); if (!Number.isInteger(parsed) || parsed < 1) { console.warn(`Ignoring invalid ${name} query param: ${raw}`); @@ -102,11 +108,19 @@ function readPositiveIntSearchParam(params: URLSearchParams, name: string): numb function getDecryptReadTuningFromLocation(): DecryptReadTuning { const params = new URLSearchParams(window.location.search); - const segmentBatchSize = readPositiveIntSearchParam(params, 'segmentBatchSize'); - const maxConcurrentSegmentBatches = readPositiveIntSearchParam( + const parsedSegmentBatchSize = readPositiveIntSearchParam(params, 'segmentBatchSize'); + const parsedMaxConcurrentSegmentBatches = readPositiveIntSearchParam( params, 'maxConcurrentSegmentBatches' ); + const segmentBatchSize = + parsedSegmentBatchSize !== undefined + ? Math.min(parsedSegmentBatchSize, MAX_SEGMENT_BATCH_SIZE) + : undefined; + const maxConcurrentSegmentBatches = + parsedMaxConcurrentSegmentBatches !== undefined + ? Math.min(parsedMaxConcurrentSegmentBatches, MAX_CONCURRENT_SEGMENT_BATCHES) + : undefined; return { ...(segmentBatchSize !== undefined && { segmentBatchSize }),