diff --git a/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json new file mode 100644 index 00000000000..815d03eee71 --- /dev/null +++ b/common/changes/@rushstack/node-core-library/fix-async-concurrency_2022-05-10-00-46.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/node-core-library", + "comment": "Fix and issue where Async.forEachAsync with an async iterator can overflow the max concurrency", + "type": "patch" + } + ], + "packageName": "@rushstack/node-core-library" +} \ No newline at end of file diff --git a/libraries/node-core-library/src/Async.ts b/libraries/node-core-library/src/Async.ts index 5b1bfa27857..194ec1a61ae 100644 --- a/libraries/node-core-library/src/Async.ts +++ b/libraries/node-core-library/src/Async.ts @@ -100,12 +100,14 @@ export class Async { async function queueOperationsAsync(): Promise { while (operationsInProgress < concurrency && !iteratorIsComplete && !promiseHasResolvedOrRejected) { + // Increment the concurrency while waiting for the iterator. + // This function is reentrant, so this ensures that at most `concurrency` executions are waiting + operationsInProgress++; const currentIteratorResult: IteratorResult = await iterator.next(); // eslint-disable-next-line require-atomic-updates iteratorIsComplete = !!currentIteratorResult.done; if (!iteratorIsComplete) { - operationsInProgress++; Promise.resolve(callback(currentIteratorResult.value, arrayIndex++)) .then(async () => { operationsInProgress--; @@ -115,6 +117,9 @@ export class Async { promiseHasResolvedOrRejected = true; reject(error); }); + } else { + // The iterator is complete and there wasn't a value, so untrack the waiting state. + operationsInProgress--; } }