Add pagination.stackAllItems option (#1214)

readme.md

@@ -791,6 +791,15 @@ The maximum amount of request that should be triggered. [Retries on failure](#re
 
 For example, it can be helpful during development to avoid an infinite number of requests.
 
+###### pagination.stackAllItems
+
+Type: `boolean`\
+Default: `true`
+
+Defines how the parameter `allItems` in [pagination.paginate](#pagination.paginate), [pagination.filter](#pagination.filter) and [pagination.shouldContinue](#pagination.shouldContinue) is managed. When set to `false`, the parameter `allItems` is always an empty array.
+
+This option can be helpful to save on memory usage when working with a large dataset.
+
 ##### localAddress
 
 Type: `string`

source/as-promise/types.ts

@@ -78,6 +78,7 @@ export interface PaginationOptions<T, R> {
 		shouldContinue?: (item: T, allItems: T[], currentItems: T[]) => boolean;
 		countLimit?: number;
 		requestLimit?: number;
+		stackAllItems?: boolean;
 	};
 }
 

source/create.ts

@@ -205,6 +205,7 @@ const create = (defaults: InstanceDefaults): Got => {
 		}
 
 		const all: T[] = [];
+		let {countLimit} = pagination;
 
 		let numberOfRequests = 0;
 		while (numberOfRequests < pagination.requestLimit) {
@@ -224,10 +225,13 @@ const create = (defaults: InstanceDefaults): Got => {
 
 					yield item;
 
-					all.push(item as T);
+					if (pagination.stackAllItems) {
+						all.push(item as T);
+					}
+
 					current.push(item as T);
 
-					if (all.length === pagination.countLimit) {
+					if (--countLimit <= 0) {
 						return;
 					}
 				}

source/index.ts

@@ -109,7 +109,8 @@ const defaults: InstanceDefaults = {
 			filter: () => true,
 			shouldContinue: () => true,
 			countLimit: Infinity,
-			requestLimit: 10000
+			requestLimit: 10000,
+			stackAllItems: true
 		}
 	},
 	handlers: [defaultHandler],

test/pagination.ts

@@ -350,3 +350,59 @@ test('`requestLimit` works', withServer, async (t, server, got) => {
 
 	t.deepEqual(results, [1]);
 });
+
+test('`stackAllItems` set to true', withServer, async (t, server, got) => {
+	attachHandler(server, 3);
+
+	let itemCount = 0;
+	const result = await got.paginate.all<number>({
+		pagination: {
+			stackAllItems: true,
+			filter: (_item, allItems, _currentItems) => {
+				t.is(allItems.length, itemCount);
+
+				return true;
+			},
+			shouldContinue: (_item, allItems, _currentItems) => {
+				t.is(allItems.length, itemCount);
+
+				return true;
+			},
+			paginate: (response, allItems, currentItems) => {
+				itemCount += 1;
+				t.is(allItems.length, itemCount);
+
+				return got.defaults.options.pagination!.paginate(response, allItems, currentItems);
+			}
+		}
+	});
+
+	t.deepEqual(result, [1, 2, 3]);
+});
+
+test('`stackAllItems` set to false', withServer, async (t, server, got) => {
+	attachHandler(server, 3);
+
+	const result = await got.paginate.all<number>({
+		pagination: {
+			stackAllItems: false,
+			filter: (_item, allItems, _currentItems) => {
+				t.is(allItems.length, 0);
+
+				return true;
+			},
+			shouldContinue: (_item, allItems, _currentItems) => {
+				t.is(allItems.length, 0);
+
+				return true;
+			},
+			paginate: (response, allItems, currentItems) => {
+				t.is(allItems.length, 0);
+
+				return got.defaults.options.pagination!.paginate(response, allItems, currentItems);
+			}
+		}
+	});
+
+	t.deepEqual(result, [1, 2, 3]);
+});