diff --git a/src/core/metrics/calculateOutputMetrics.ts b/src/core/metrics/calculateOutputMetrics.ts index 9b155c0c2..3c2bafcd6 100644 --- a/src/core/metrics/calculateOutputMetrics.ts +++ b/src/core/metrics/calculateOutputMetrics.ts @@ -1,11 +1,8 @@ import { logger } from '../../shared/logger.js'; +import { getProcessConcurrency } from '../../shared/processConcurrency.js'; import { type MetricsTaskRunner, runTokenCount } from './metricsWorkerRunner.js'; import type { TokenEncoding } from './TokenCounter.js'; -// Target ~200K characters per chunk to balance tokenization throughput and worker round-trip overhead. -// Benchmarks show 200K is the sweet spot: fewer round-trips than 100K with enough chunks -// for good parallelism across available threads (e.g., 20 chunks for a 4M character output). -const TARGET_CHARS_PER_CHUNK = 200_000; const MIN_CONTENT_LENGTH_FOR_PARALLEL = 1_000_000; // 1MB export const calculateOutputMetrics = async ( @@ -23,11 +20,15 @@ export const calculateOutputMetrics = async ( let result: number; if (shouldRunInParallel) { - // Split content into chunks for parallel processing + // Split content into chunks matching the number of available CPU cores. + // Using fewer, larger chunks minimizes IPC overhead (postMessage serialization + // per task) while still saturating all worker threads. + const numChunks = Math.max(1, getProcessConcurrency()); + const chunkSize = Math.ceil(content.length / numChunks); const chunks: string[] = []; - for (let i = 0; i < content.length; i += TARGET_CHARS_PER_CHUNK) { - chunks.push(content.slice(i, i + TARGET_CHARS_PER_CHUNK)); + for (let i = 0; i < content.length; i += chunkSize) { + chunks.push(content.slice(i, i + chunkSize)); } // Process chunks in parallel diff --git a/src/shared/memoryUtils.ts b/src/shared/memoryUtils.ts index fbebe8e50..4b5933701 100644 --- a/src/shared/memoryUtils.ts +++ b/src/shared/memoryUtils.ts @@ -2,7 +2,7 @@ * Memory utility functions for monitoring memory usage across the application */ -import { logger } from './logger.js'; +import { logger, repomixLogLevels } from './logger.js'; export interface MemoryStats { heapUsed: number; @@ -44,6 +44,7 @@ export function getMemoryStats(): MemoryStats { * Log memory usage at trace level with a context message */ export function logMemoryUsage(context: string): void { + if (logger.getLogLevel() < repomixLogLevels.DEBUG) return; const stats = getMemoryStats(); logger.trace( `Memory [${context}] | Heap: ${stats.heapUsed}/${stats.heapTotal}MB (${stats.heapUsagePercent}%) | RSS: ${stats.rss}MB | Ext: ${stats.external}MB`, @@ -69,6 +70,10 @@ export function logMemoryDifference(context: string, before: MemoryStats, after: * Execute a function and log memory usage before and after */ export async function withMemoryLogging(context: string, fn: () => Promise): Promise { + if (logger.getLogLevel() < repomixLogLevels.DEBUG) { + return fn(); + } + const before = getMemoryStats(); logMemoryUsage(`${context} - Before`); diff --git a/tests/core/metrics/calculateOutputMetrics.test.ts b/tests/core/metrics/calculateOutputMetrics.test.ts index f3ec9f81a..15eff8285 100644 --- a/tests/core/metrics/calculateOutputMetrics.test.ts +++ b/tests/core/metrics/calculateOutputMetrics.test.ts @@ -7,7 +7,7 @@ import { type TokenCountTask, } from '../../../src/core/metrics/workers/calculateMetricsWorker.js'; import { logger } from '../../../src/shared/logger.js'; -import type { WorkerOptions } from '../../../src/shared/processConcurrency.js'; +import { getProcessConcurrency, type WorkerOptions } from '../../../src/shared/processConcurrency.js'; vi.mock('../../../src/shared/logger'); @@ -118,8 +118,9 @@ describe('calculateOutputMetrics', () => { taskRunner: mockParallelTaskRunner({ numOfTasks: 1, workerType: 'calculateMetrics', runtime: 'worker_threads' }), }); - expect(chunksProcessed).toBeGreaterThan(1); // Should have processed multiple chunks - expect(result).toBe(chunksProcessed * 100); // chunks * 100 tokens per chunk + const expectedChunks = getProcessConcurrency(); + expect(chunksProcessed).toBe(expectedChunks); // Should match number of CPU cores + expect(result).toBe(expectedChunks * 100); // numChunks * 100 tokens per chunk }); it('should handle errors in parallel processing', async () => { @@ -173,14 +174,14 @@ describe('calculateOutputMetrics', () => { }), }); - // With TARGET_CHARS_PER_CHUNK=200_000, 1.1M character content should produce 6 chunks + // Check that chunks are roughly equal in size + const expectedChunks = getProcessConcurrency(); const chunkSizes = processedChunks.map((chunk) => chunk.length); - expect(processedChunks.length).toBe(6); - // All chunks except the last should be exactly TARGET_CHARS_PER_CHUNK - for (let i = 0; i < chunkSizes.length - 1; i++) { - expect(chunkSizes[i]).toBe(200_000); - } + expect(processedChunks.length).toBe(expectedChunks); // Should match number of CPU cores + // Last chunk may be smaller due to Math.ceil rounding + const maxDiff = Math.max(...chunkSizes) - Math.min(...chunkSizes); + expect(maxDiff).toBeLessThan(Math.ceil(content.length / expectedChunks)); // Chunks should be roughly equal expect(processedChunks.join('')).toBe(content); // All content should be processed }); });