feat: optimize worker system performance

package.json

@@ -28,9 +28,11 @@
     "lint-secretlint": "secretlint \"**/*\" --secretlintignore .gitignore",
     "test": "vitest",
     "test-coverage": "vitest run --coverage",
-    "repomix": "node --run build && node --trace-warnings bin/repomix.cjs",
+    "repomix": "node --run build && node --enable-source-maps --trace-warnings bin/repomix.cjs",
     "repomix-src": "node --run repomix -- --include 'src,tests'",
     "repomix-website": "node --run repomix -- --include 'website'",
+    "time-node": "node --run build && time node bin/repomix.cjs",
+    "time-bun": "bun run build && time bun bin/repomix.cjs",
     "memory-check": "node --run repomix -- --verbose | grep Memory",
     "memory-check-one-file": "node --run repomix -- --verbose --include 'package.json' | grep Memory",
     "website": "docker compose -f website/compose.yml up --build",

src/cli/actions/defaultAction.ts

@@ -1,4 +1,3 @@
-import path from 'node:path';
 import { loadFileConfig, mergeConfigs } from '../../config/configLoad.js';
 import {
   type RepomixConfigCli,
@@ -7,16 +6,20 @@ import {
   type RepomixOutputStyle,
   repomixConfigCliSchema,
 } from '../../config/configSchema.js';
-import { readFilePathsFromStdin } from '../../core/file/fileStdin.js';
-import { type PackResult, pack } from '../../core/packager.js';
-import { RepomixError } from '../../shared/errorHandle.js';
+import type { PackResult } from '../../core/packager.js';
 import { rethrowValidationErrorIfZodError } from '../../shared/errorHandle.js';
 import { logger } from '../../shared/logger.js';
 import { splitPatterns } from '../../shared/patternUtils.js';
+import { initTaskRunner } from '../../shared/processConcurrency.js';
 import { reportResults } from '../cliReport.js';
-import { Spinner } from '../cliSpinner.js';
 import type { CliOptions } from '../types.js';
 import { runMigrationAction } from './migrationAction.js';
+import type {
+  DefaultActionTask,
+  DefaultActionWorkerResult,
+  PingResult,
+  PingTask,
+} from './workers/defaultActionWorker.js';
 
 export interface DefaultActionRunnerResult {
   packResult: PackResult;
@@ -33,7 +36,7 @@ export const runDefaultAction = async (
   // Run migration before loading config
   await runMigrationAction(cwd);
 
-  // Load the config file
+  // Load the config file in main process
   const fileConfig: RepomixConfigFile = await loadFileConfig(cwd, cliOptions.config ?? null);
   logger.trace('Loaded file config:', fileConfig);
 
@@ -45,95 +48,40 @@ export const runDefaultAction = async (
   const config: RepomixConfigMerged = mergeConfigs(cwd, fileConfig, cliConfig);
   logger.trace('Merged config:', config);
 
-  // Initialize spinner that can be shared across operations
-  const spinner = new Spinner('Initializing...', cliOptions);
-  spinner.start();
-
-  const result = cliOptions.stdin
-    ? await handleStdinProcessing(directories, cwd, config, spinner)
-    : await handleDirectoryProcessing(directories, cwd, config, spinner);
-
-  spinner.succeed('Packing completed successfully!');
-
-  const packResult = result.packResult;
-
-  reportResults(cwd, packResult, config);
-
-  return {
-    packResult,
-    config,
-  };
-};
-
-/**
- * Handles stdin processing workflow for file paths input.
- */
-export const handleStdinProcessing = async (
-  directories: string[],
-  cwd: string,
-  config: RepomixConfigMerged,
-  spinner: Spinner,
-): Promise<DefaultActionRunnerResult> => {
-  // Validate directory arguments for stdin mode
-  const firstDir = directories[0] ?? '.';
-  if (directories.length > 1 || firstDir !== '.') {
-    throw new RepomixError(
-      'When using --stdin, do not specify directory arguments. File paths will be read from stdin.',
-    );
-  }
-
-  let packResult: PackResult;
+  // Create worker task runner
+  const taskRunner = initTaskRunner<DefaultActionTask | PingTask, DefaultActionWorkerResult | PingResult>({
+    numOfTasks: 1,
+    workerPath: new URL('./workers/defaultActionWorker.js', import.meta.url).href,
+    runtime: 'child_process',
+  });
 
   try {
-    const stdinResult = await readFilePathsFromStdin(cwd);
+    // Wait for worker to be ready (Bun compatibility)
+    await waitForWorkerReady(taskRunner);
 
-    // Use pack with predefined files from stdin
-    packResult = await pack(
-      [cwd],
+    // Create task for worker (now with pre-loaded config)
+    const task: DefaultActionTask = {
+      directories,
+      cwd,
       config,
-      (message) => {
-        spinner.update(message);
-      },
-      {},
-      stdinResult.filePaths,
-    );
-  } catch (error) {
-    spinner.fail('Error reading from stdin or during packing');
-    throw error;
-  }
+      cliOptions,
+      isStdin: !!cliOptions.stdin,
+    };
 
-  return {
-    packResult,
-    config,
-  };
-};
+    // Run the task in worker (spinner is handled inside worker)
+    const result = (await taskRunner.run(task)) as DefaultActionWorkerResult;
 
-/**
- * Handles normal directory processing workflow.
- */
-export const handleDirectoryProcessing = async (
-  directories: string[],
-  cwd: string,
-  config: RepomixConfigMerged,
-  spinner: Spinner,
-): Promise<DefaultActionRunnerResult> => {
-  const targetPaths = directories.map((directory) => path.resolve(cwd, directory));
-
-  let packResult: PackResult;
+    // Report results in main process
+    reportResults(cwd, result.packResult, result.config);
 
-  try {
-    packResult = await pack(targetPaths, config, (message) => {
-      spinner.update(message);
-    });
-  } catch (error) {
-    spinner.fail('Error during packing');
-    throw error;
+    return {
+      packResult: result.packResult,
+      config: result.config,
+    };
+  } finally {
+    // Always cleanup worker pool
+    await taskRunner.cleanup();
   }
-
-  return {
-    packResult,
-    config,
-  };
 };
 
 /**
@@ -317,3 +265,42 @@ export const buildCliConfig = (options: CliOptions): RepomixConfigCli => {
     throw error;
   }
 };
+
+/**
+ * Wait for worker to be ready by sending a ping request.
+ * This is specifically needed for Bun compatibility due to ES module initialization timing issues.
+ */
+const waitForWorkerReady = async (taskRunner: {
+  run: (task: DefaultActionTask | PingTask) => Promise<DefaultActionWorkerResult | PingResult>;
+}): Promise<void> => {
+  const isBun = process.versions?.bun;
+  if (!isBun) {
+    // No need to wait for Node.js
+    return;
+  }
+
+  const maxRetries = 3;
+  const retryDelay = 50; // ms
+  let pingSuccessful = false;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      await taskRunner.run({
+        ping: true,
+      });
+      logger.debug(`Worker initialization ping successful on attempt ${attempt}`);
+      pingSuccessful = true;
+      break;
+    } catch (error) {
+      logger.debug(`Worker ping failed on attempt ${attempt}/${maxRetries}:`, error);
+      if (attempt < maxRetries) {
+        logger.debug(`Waiting ${retryDelay}ms before retry...`);
+        await new Promise((resolve) => setTimeout(resolve, retryDelay));
+      }
+    }
+  }
+
+  if (!pingSuccessful) {
+    logger.debug('All Worker ping attempts failed, proceeding anyway...');
+  }
+};

src/cli/actions/workers/defaultActionWorker.ts

@@ -0,0 +1,109 @@
+import path from 'node:path';
+import type { RepomixConfigMerged } from '../../../config/configSchema.js';
+import { readFilePathsFromStdin } from '../../../core/file/fileStdin.js';
+import { type PackResult, pack } from '../../../core/packager.js';
+import { RepomixError } from '../../../shared/errorHandle.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
+import { Spinner } from '../../cliSpinner.js';
+import type { CliOptions } from '../../types.js';
+
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
+
+export interface DefaultActionTask {
+  directories: string[];
+  cwd: string;
+  config: RepomixConfigMerged;
+  cliOptions: CliOptions;
+  isStdin: boolean;
+}
+
+export interface PingTask {
+  ping: true;
+}
+
+export interface DefaultActionWorkerResult {
+  packResult: PackResult;
+  config: RepomixConfigMerged;
+}
+
+export interface PingResult {
+  ping: true;
+}
+
+// Function overloads for better type inference
+function defaultActionWorker(task: DefaultActionTask): Promise<DefaultActionWorkerResult>;
+function defaultActionWorker(task: PingTask): Promise<PingResult>;
+async function defaultActionWorker(
+  task: DefaultActionTask | PingTask,
+): Promise<DefaultActionWorkerResult | PingResult> {
+  // Handle ping requests for Bun compatibility check
+  if ('ping' in task) {
+    return {
+      ping: true,
+    };
+  }
+
+  // At this point, task is guaranteed to be DefaultActionTask
+  const { directories, cwd, config, cliOptions, isStdin } = task;
+
+  logger.trace('Worker: Using pre-loaded config:', config);
+
+  // Initialize spinner in worker
+  const spinner = new Spinner('Initializing...', cliOptions);
+  spinner.start();
+
+  let packResult: PackResult;
+
+  try {
+    if (isStdin) {
+      // Handle stdin processing
+      // Validate directory arguments for stdin mode
+      const firstDir = directories[0] ?? '.';
+      if (directories.length > 1 || firstDir !== '.') {
+        throw new RepomixError(
+          'When using --stdin, do not specify directory arguments. File paths will be read from stdin.',
+        );
+      }
+
+      const stdinResult = await readFilePathsFromStdin(cwd);
+
+      // Use pack with predefined files from stdin
+      packResult = await pack(
+        [cwd],
+        config,
+        (message) => {
+          spinner.update(message);
+        },
+        {},
+        stdinResult.filePaths,
+      );
+    } else {
+      // Handle directory processing
+      const targetPaths = directories.map((directory) => path.resolve(cwd, directory));
+
+      packResult = await pack(targetPaths, config, (message) => {
+        spinner.update(message);
+      });
+    }
+
+    spinner.succeed('Packing completed successfully!');
+
+    return {
+      packResult,
+      config,
+    };
+  } catch (error) {
+    spinner.fail('Error during packing');
+    throw error;
+  }
+}
+
+export default defaultActionWorker;
+
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = async () => {
+  // Any cleanup needed when worker terminates
+  // Currently no specific cleanup required for defaultAction worker
+};

src/core/file/fileProcess.ts

@@ -25,7 +25,7 @@ export const processFiles = async (
     numOfTasks: rawFiles.length,
     workerPath: new URL('./workers/fileProcessWorker.js', import.meta.url).href,
     // High memory usage and leak risk
-    runtime: 'child_process',
+    runtime: 'worker_threads',
   });
   const tasks = rawFiles.map(
     (rawFile, _index) =>

src/core/file/fileSearch.ts

@@ -1,13 +1,14 @@
 import type { Stats } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { globby } from 'globby';
 import { minimatch } from 'minimatch';
 import type { RepomixConfigMerged } from '../../config/configSchema.js';
 import { defaultIgnoreList } from '../../config/defaultIgnore.js';
 import { RepomixError } from '../../shared/errorHandle.js';
 import { logger } from '../../shared/logger.js';
 import { sortPaths } from './filePathSort.js';
-import { executeGlobbyInWorker } from './globbyExecute.js';
+
 import { PermissionError, checkDirectoryPermissions } from './permissionCheck.js';
 
 export interface FileSearchResult {
@@ -191,17 +192,18 @@ export const searchFiles = async (
 
     logger.trace('Include patterns with explicit files:', includePatterns);
 
-    const filePaths = await executeGlobbyInWorker(includePatterns, {
+    const filePaths = await globby(includePatterns, {
       cwd: rootDir,
       ignore: [...adjustedIgnorePatterns],
       ignoreFiles: [...ignoreFilePatterns],
       onlyFiles: true,
       absolute: false,
       dot: true,
       followSymbolicLinks: false,
-    }).catch((error) => {
+    }).catch((error: unknown) => {
       // Handle EPERM errors specifically
-      if (error.code === 'EPERM' || error.code === 'EACCES') {
+      const code = (error as NodeJS.ErrnoException | { code?: string })?.code;
+      if (code === 'EPERM' || code === 'EACCES') {
         throw new PermissionError(
           `Permission denied while scanning directory. Please check folder access permissions for your terminal app. path: ${rootDir}`,
           rootDir,
@@ -212,7 +214,7 @@ export const searchFiles = async (
 
     let emptyDirPaths: string[] = [];
     if (config.output.includeEmptyDirectories) {
-      const directories = await executeGlobbyInWorker(includePatterns, {
+      const directories = await globby(includePatterns, {
         cwd: rootDir,
         ignore: [...adjustedIgnorePatterns],
         ignoreFiles: [...ignoreFilePatterns],