From dab9f417aaca2bd6cf87a6983275a47c8c3f20a5 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 9 May 2022 17:46:59 -0700 Subject: [PATCH 1/2] [node-core-library] Fix Async.forEachAsync concurrency --- .../fix-async-concurrency_2022-05-10-00-46.json | 10 ++++++++++ libraries/node-core-library/src/Async.ts | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json diff --git a/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json new file mode 100644 index 00000000000..84c032404bc --- /dev/null +++ b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Fix Async.forEachAsync with an async iterator overflowing the max concurrency.", + "type": "patch" + } + ], + "packageName": "@rushstack/node-core-library" +} \ No newline at end of file diff --git a/libraries/node-core-library/src/Async.ts b/libraries/node-core-library/src/Async.ts index 5b1bfa27857..194ec1a61ae 100644 --- a/libraries/node-core-library/src/Async.ts +++ b/libraries/node-core-library/src/Async.ts @@ -100,12 +100,14 @@ export class Async { async function queueOperationsAsync(): Promise { while (operationsInProgress < concurrency && !iteratorIsComplete && !promiseHasResolvedOrRejected) { + // Increment the concurrency while waiting for the iterator. + // This function is reentrant, so this ensures that at most `concurrency` executions are waiting + operationsInProgress++; const currentIteratorResult: IteratorResult = await iterator.next(); // eslint-disable-next-line require-atomic-updates iteratorIsComplete = !!currentIteratorResult.done; if (!iteratorIsComplete) { - operationsInProgress++; Promise.resolve(callback(currentIteratorResult.value, arrayIndex++)) .then(async () => { operationsInProgress--; @@ -115,6 +117,9 @@ export class Async { promiseHasResolvedOrRejected = true; reject(error); }); + } else { + // The iterator is complete and there wasn't a value, so untrack the waiting state. + operationsInProgress--; } } From 4ba5ccabfb2e5f1252dcec43fc7f04829175599f Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Mon, 9 May 2022 17:54:32 -0700 Subject: [PATCH 2/2] Rush change. --- .../fix-async-concurrency_2022-05-10-00-46.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json index 84c032404bc..815d03eee71 100644 --- a/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json +++ b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/node-core-library", - "comment": "Fix Async.forEachAsync with an async iterator overflowing the max concurrency.", + "comment": "Fix and issue where Async.forEachAsync with an async iterator can overflow the max concurrency", "type": "patch" } ],