Implement --puppeteer-page-pool-max

README.md

@@ -86,6 +86,7 @@ kill $PID
       --help                       Shows this help message
       --puppeteer-launch-mode      Options: launch, connect. Default: launch
       --puppeteer-launch-options   Puppeteer .launch or .connect options in JS. Default: {"headless":true}
+      --puppeteer-page-pool-max    Maximum pages to use at the same time with Puppeteer. Default: 10
       --include-hash               When enabled, URL hashes are not ignored when crawling. Default: false
       --include-search-query       When enabled, URL search queries are not ignored when crawling. Default: false
       --should-visit               Custom JS function that can limit which links the crawler follows.

src/cli.ts

@@ -40,6 +40,9 @@ export function formatHelp(defaultConfig: typeof _defaultConfig) {
       --puppeteer-launch-options   Puppeteer .launch or .connect options in JS. Default: ${JSON.stringify(
         defaultConfig.puppeteerLaunchMode.options
       )}
+      --puppeteer-page-pool-max    Maximum pages to use at the same time with Puppeteer. Default: ${
+        defaultConfig.puppeteerPagePoolMax
+      }
       --include-hash               When enabled, URL hashes are not ignored when crawling. Default: ${
         defaultConfig.includeHash
       }
@@ -132,6 +135,7 @@ export function parseCliArgs() {
     '--max-depth': Number,
     '--puppeteer-launch-mode': String,
     '--puppeteer-launch-options': String,
+    '--puppeteer-page-pool-max': Number,
     '--after-goto': String,
     '--after-page': String,
     '--single-page': Boolean,

src/config.ts

@@ -21,6 +21,7 @@ export type Config = {
   shouldVisit?: string
   trailingSlashMode: 'preserve' | 'remove' | 'add'
   puppeteerLaunchMode: puppeteerLaunchMode
+  puppeteerPagePoolMax: number
   maxDepth: number
   outDir: string
   outFile: string
@@ -55,6 +56,7 @@ export const defaultConfig = {
       headless: true,
     },
   },
+  puppeteerPagePoolMax: 10,
   outDir: '.squint',
   screenshotOptions: {},
 }
@@ -98,6 +100,8 @@ export function parseConfig() {
         args['--puppeteer-launch-options'] ??
         defaultConfig.puppeteerLaunchMode.options,
     } as Config['puppeteerLaunchMode'],
+    puppeteerPagePoolMax:
+      args['--puppeteer-page-pool-max'] ?? defaultConfig.puppeteerPagePoolMax,
     afterGoto: args['--after-goto'],
     afterPage: args['--after-page'],
     selector: args['--selector'],

src/puppeteer.ts

@@ -19,7 +19,7 @@ export function createPagePool(browser: Browser, config: Config) {
   }
 
   return genericPool.createPool(factory, {
-    max: 10,
+    max: config.puppeteerPagePoolMax,
     min: 1,
   })
 }