feat(core): Implement worker memory management with Tinypool teardown and reorganize memory testing tools

benchmarks/memory/.gitignore

@@ -7,6 +7,7 @@ dist/
 # Test outputs
 memory-test-output.txt
 test-output-*.txt
+test-output.txt
 
 # Test results
 memory-history-*.json
@@ -20,4 +21,4 @@ npm-debug.log*
 
 # OS files
 .DS_Store
-Thumbs.db
+Thumbs.db

benchmarks/memory/package.json

@@ -14,6 +14,8 @@
     "test:continuous": "node --run build:all && node --expose-gc dist/memory-test.js --continuous"
   },
   "dependencies": {
+    "@types/asciichart": "^1.5.8",
+    "asciichart": "^1.5.25",
     "repomix": "file:../.."
   },
   "devDependencies": {

benchmarks/memory/src/memory-test.ts

@@ -9,6 +9,7 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
+import * as asciichart from 'asciichart';
 import { runCli } from 'repomix';
 import type { MemoryHistory, MemoryTestSummary, MemoryUsage, TestConfig } from './types.js';
 
@@ -22,6 +23,7 @@ const flags = {
   continuous: args.includes('--continuous'),
   saveResults: args.includes('--save') || args.includes('-s'),
   help: args.includes('--help') || args.includes('-h'),
+  showGraph: args.includes('--graph') || args.includes('-g'),
 };
 
 // Extract numeric arguments
@@ -31,7 +33,7 @@ const delay = Number(numericArgs[1]) || (flags.full ? 100 : 50);
 
 // Configuration
 const MEMORY_LOG_INTERVAL = flags.full ? 10 : 5;
-const FORCE_GC_INTERVAL = flags.full ? 20 : 10;
+const FORCE_GC_INTERVAL = flags.full ? 50 : 20;
 const WARNING_THRESHOLD = flags.full ? 50 : 100; // Memory growth percentage
 
 // Test configuration
@@ -42,7 +44,7 @@ const TEST_CONFIG: TestConfig = {
   options: {
     include: 'src/**/*.ts',
     output: path.join(__dirname, '../test-output.txt'),
-    style: 'plain',
+    compress: true,
     quiet: true,
   },
 };
@@ -62,6 +64,7 @@ Options:
   --full, -f       Enable comprehensive testing (more iterations, detailed analysis)
   --continuous     Run until stopped with Ctrl+C
   --save, -s       Save detailed results to JSON file
+  --graph, -g      Show real-time ASCII graphs during testing
   --help, -h       Show this help message
 
 Examples:
@@ -92,9 +95,7 @@ function getMemoryUsage(): MemoryUsage {
 function forceGC(): void {
   if (global.gc) {
     global.gc();
-    if (flags.full) {
-      console.log('🗑️  Forced garbage collection');
-    }
+    console.log('🗑️  Forced garbage collection');
   }
 }
 
@@ -113,10 +114,18 @@ function logMemoryUsage(iteration: number, configName: string, error: Error | nu
   const statusIcon = error ? '❌' : '✅';
   const errorText = error ? ` (ERROR: ${error.message})` : '';
 
+  // Format with fixed widths for alignment
+  const iterationStr = `Iteration ${iteration.toString().padStart(3)}`;
+  const configStr = configName.padEnd(12);
+  const heapStr = `${usage.heapUsed.toString().padStart(6)}MB`;
+  const heapTotalStr = `${usage.heapTotal.toString().padStart(6)}MB`;
+  const heapPercentStr = `(${usage.heapUsagePercent.toString().padStart(5)}%)`;
+  const rssStr = `${usage.rss.toString().padStart(6)}MB`;
+
   console.log(
-    `${statusIcon} Iteration ${iteration}: ${configName} - ` +
-      `Heap: ${usage.heapUsed}MB/${usage.heapTotal}MB (${usage.heapUsagePercent}%), ` +
-      `RSS: ${usage.rss}MB${errorText}`,
+    `${statusIcon} ${iterationStr}: ${configStr} - ` +
+      `Heap: ${heapStr}/${heapTotalStr} ${heapPercentStr}, ` +
+      `RSS: ${rssStr}${errorText}`,
   );
 }
 
@@ -130,6 +139,33 @@ async function cleanupFiles(): Promise<void> {
   }
 }
 
+function displayMemoryGraphs(): void {
+  if (memoryHistory.length < 5 || !flags.showGraph) return;
+
+  const recentHistory = memoryHistory.slice(-40); // Last 40 data points for graph
+
+  const heapData = recentHistory.map((entry) => entry.heapUsed);
+  const rssData = recentHistory.map((entry) => entry.rss);
+
+  console.log('\n📈 Memory Usage Graphs:');
+
+  console.log('\n🔸 Heap Usage (MB):');
+  console.log(
+    asciichart.plot(heapData, {
+      height: 8,
+      format: (x: number) => x.toFixed(1),
+    }),
+  );
+
+  console.log('\n🔹 RSS Usage (MB):');
+  console.log(
+    asciichart.plot(rssData, {
+      height: 8,
+      format: (x: number) => x.toFixed(1),
+    }),
+  );
+}
+
 function analyzeMemoryTrends(): void {
   if (memoryHistory.length < 10) return;
 
@@ -153,6 +189,9 @@ function analyzeMemoryTrends(): void {
   if (heapGrowth > WARNING_THRESHOLD || rssGrowth > WARNING_THRESHOLD) {
     console.log('⚠️  WARNING: Significant memory growth detected - possible memory leak!');
   }
+
+  // Show graphs if enabled
+  displayMemoryGraphs();
 }
 
 async function saveMemoryHistory(): Promise<void> {
@@ -263,6 +302,12 @@ async function runMemoryTest(): Promise<void> {
     }
   }
 
+  // Show final graph if requested
+  if (flags.showGraph && memoryHistory.length >= 5) {
+    console.log('\n📈 Complete Memory Usage Timeline:');
+    displayMemoryGraphs();
+  }
+
   // Save results if requested
   await saveMemoryHistory();
 
@@ -307,7 +352,12 @@ console.log('🧪 Memory Test');
 console.log(`📋 Mode: ${flags.full ? 'Comprehensive' : 'Basic'} (${iterations} iterations, ${delay}ms delay)`);
 console.log(
   `⚡ Features: ${
-    [flags.continuous && 'Continuous Mode', flags.saveResults && 'Save Results', flags.full && 'Full Analysis']
+    [
+      flags.continuous && 'Continuous Mode',
+      flags.saveResults && 'Save Results',
+      flags.full && 'Full Analysis',
+      flags.showGraph && 'Graph Display',
+    ]
       .filter(Boolean)
       .join(', ') || 'Basic Test'
   }`,

benchmarks/memory/src/types.ts

@@ -26,7 +26,6 @@ export interface TestConfig {
     include?: string;
     ignore?: string;
     output: string;
-    style: 'plain' | 'xml' | 'markdown';
     compress?: boolean;
     quiet: boolean;
   };

src/core/file/fileCollect.ts

@@ -26,7 +26,6 @@ export const collectFiles = async (
   const taskRunner = deps.initTaskRunner<FileCollectTask, FileCollectResult>({
     numOfTasks: filePaths.length,
     workerPath: new URL('./workers/fileCollectWorker.js', import.meta.url).href,
-    // Use worker_threads for file collection - low memory leak risk
     runtime: 'worker_threads',
   });
   const tasks = filePaths.map(

src/core/file/fileProcess.ts

@@ -24,6 +24,8 @@ export const processFiles = async (
   const taskRunner = deps.initTaskRunner<FileProcessTask, ProcessedFile>({
     numOfTasks: rawFiles.length,
     workerPath: new URL('./workers/fileProcessWorker.js', import.meta.url).href,
+    // High memory usage and leak risk
+    runtime: 'child_process',
   });
   const tasks = rawFiles.map(
     (rawFile, _index) =>

src/core/file/globbyExecute.ts

@@ -16,6 +16,7 @@ export const executeGlobbyInWorker = async (
   const taskRunner = deps.initTaskRunner<GlobbyTask, string[]>({
     numOfTasks: 1,
     workerPath: new URL('./workers/globbyWorker.js', import.meta.url).href,
+    runtime: 'worker_threads',
   });
 
   try {

src/core/file/workers/fileCollectWorker.ts

@@ -49,3 +49,8 @@ export default async ({ filePath, rootDir, maxFileSize }: FileCollectTask): Prom
     `File processing for ${filePath} resulted in an unexpected state: content is null but no skip reason was provided.`,
   );
 };
+
+// Export cleanup function for Tinypool teardown (no cleanup needed for this worker)
+export const onWorkerTermination = () => {
+  // No cleanup needed for file collection worker
+};

src/core/file/workers/fileProcessWorker.ts

@@ -1,5 +1,6 @@
 import type { RepomixConfigMerged } from '../../../config/configSchema.js';
 import { setLogLevelByWorkerData } from '../../../shared/logger.js';
+import { cleanupLanguageParser } from '../../treeSitter/parseFile.js';
 import { processContent } from '../fileProcessContent.js';
 import type { ProcessedFile, RawFile } from '../fileTypes.js';
 
@@ -19,3 +20,8 @@ export default async ({ rawFile, config }: FileProcessTask): Promise<ProcessedFi
     content: processedContent,
   };
 };
+
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = async () => {
+  await cleanupLanguageParser();
+};

src/core/file/workers/globbyWorker.ts

@@ -8,3 +8,8 @@ export interface GlobbyTask {
 export default async ({ patterns, options }: GlobbyTask): Promise<string[]> => {
   return globby(patterns, options);
 };
+
+// Export cleanup function for Tinypool teardown (no cleanup needed for this worker)
+export const onWorkerTermination = () => {
+  // No cleanup needed for globby worker
+};

src/core/metrics/calculateGitDiffMetrics.ts

@@ -26,6 +26,7 @@ export const calculateGitDiffMetrics = async (
   const taskRunner = deps.initTaskRunner<GitDiffMetricsTask, number>({
     numOfTasks: 1, // Single task for git diff calculation
     workerPath: new URL('./workers/gitDiffMetricsWorker.js', import.meta.url).href,
+    runtime: 'child_process',
   });
 
   try {

src/core/metrics/calculateGitLogMetrics.ts

@@ -31,6 +31,7 @@ export const calculateGitLogMetrics = async (
   const taskRunner = deps.initTaskRunner<GitLogMetricsTask, number>({
     numOfTasks: 1, // Single task for git log calculation
     workerPath: new URL('./workers/gitLogMetricsWorker.js', import.meta.url).href,
+    runtime: 'child_process',
   });
 
   try {

src/core/metrics/calculateOutputMetrics.ts

@@ -19,6 +19,7 @@ export const calculateOutputMetrics = async (
   const taskRunner = deps.initTaskRunner<OutputMetricsTask, number>({
     numOfTasks,
     workerPath: new URL('./workers/outputMetricsWorker.js', import.meta.url).href,
+    runtime: 'child_process',
   });
 
   try {

src/core/metrics/calculateSelectiveFileMetrics.ts

@@ -26,6 +26,7 @@ export const calculateSelectiveFileMetrics = async (
   const taskRunner = deps.initTaskRunner<FileMetricsTask, FileMetrics>({
     numOfTasks: filesToProcess.length,
     workerPath: new URL('./workers/fileMetricsWorker.js', import.meta.url).href,
+    runtime: 'child_process',
   });
   const tasks = filesToProcess.map(
     (file, index) =>

src/core/metrics/workers/fileMetricsWorker.ts

@@ -37,7 +37,7 @@ export const calculateIndividualFileMetrics = async (
   return { path: file.path, charCount, tokenCount };
 };
 
-// Cleanup when worker is terminated
-process.on('exit', () => {
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = () => {
   freeTokenCounters();
-});
+};

src/core/metrics/workers/gitDiffMetricsWorker.ts

@@ -36,7 +36,7 @@ export default async ({ workTreeDiffContent, stagedDiffContent, encoding }: GitD
   return totalTokens;
 };
 
-// Cleanup when worker is terminated
-process.on('exit', () => {
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = () => {
   freeTokenCounters();
-});
+};

src/core/metrics/workers/gitLogMetricsWorker.ts

@@ -30,8 +30,10 @@ export default async ({ content, encoding }: GitLogMetricsTask): Promise<number>
   } catch (error) {
     logger.error('Error calculating git log token count:', error);
     return 0;
-  } finally {
-    // Clean up token counters to free memory
-    freeTokenCounters();
   }
 };
+
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = () => {
+  freeTokenCounters();
+};

src/core/metrics/workers/outputMetricsWorker.ts

@@ -25,7 +25,7 @@ export default async ({ content, encoding, path }: OutputMetricsTask): Promise<n
   return tokenCount;
 };
 
-// Cleanup when worker is terminated
-process.on('exit', () => {
+// Export cleanup function for Tinypool teardown
+export const onWorkerTermination = () => {
   freeTokenCounters();
-});
+};

src/core/security/securityCheck.ts

@@ -58,6 +58,8 @@ export const runSecurityCheck = async (
   const taskRunner = deps.initTaskRunner<SecurityCheckTask, SuspiciousFileResult | null>({
     numOfTasks: rawFiles.length + gitDiffTasks.length + gitLogTasks.length,
     workerPath: new URL('./workers/securityCheckWorker.js', import.meta.url).href,
+    // Low memory leak risk
+    runtime: 'worker_threads',
   });
   const fileTasks = rawFiles.map(
     (file) =>

src/core/security/workers/securityCheckWorker.ts

@@ -83,3 +83,8 @@ export const createSecretLintConfig = (): SecretLintCoreConfig => ({
     },
   ],
 });
+
+// Export cleanup function for Tinypool teardown (no cleanup needed for this worker)
+export const onWorkerTermination = () => {
+  // No cleanup needed for security check worker
+};