feat(core): Improve error handling and performance optimizations

src/core/file/workers/fileCollectWorker.ts

@@ -1,16 +1,17 @@
 import path from 'node:path';
-import { logger, setLogLevelByEnv } from '../../../shared/logger.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
 import { readRawFile } from '../fileRead.js';
 
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
+
 export interface FileCollectTask {
   filePath: string;
   rootDir: string;
   maxFileSize: number;
 }
 
-// Set logger log level from environment variable if provided
-setLogLevelByEnv();
-
 export default async ({ filePath, rootDir, maxFileSize }: FileCollectTask) => {
   const fullPath = path.resolve(rootDir, filePath);
   const content = await readRawFile(fullPath, maxFileSize);

src/core/file/workers/fileProcessWorker.ts

@@ -1,16 +1,17 @@
 import type { RepomixConfigMerged } from '../../../config/configSchema.js';
-import { logger, setLogLevelByEnv } from '../../../shared/logger.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
 import { processContent } from '../fileProcessContent.js';
 import type { ProcessedFile, RawFile } from '../fileTypes.js';
 
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
+
 export interface FileProcessTask {
   rawFile: RawFile;
   config: RepomixConfigMerged;
 }
 
-// Set logger log level from environment variable if provided
-setLogLevelByEnv();
-
 export default async ({ rawFile, config }: FileProcessTask): Promise<ProcessedFile> => {
   const processedContent = await processContent(rawFile, config);
   return {

src/core/metrics/TokenCounter.ts

@@ -5,8 +5,15 @@ export class TokenCounter {
   private encoding: Tiktoken;
 
   constructor(encodingName: TiktokenEncoding) {
+    const startTime = process.hrtime.bigint();
+
     // Setup encoding with the specified model
     this.encoding = get_encoding(encodingName);
+
+    const endTime = process.hrtime.bigint();
+    const initTime = Number(endTime - startTime) / 1e6; // Convert to milliseconds
+
+    logger.debug(`TokenCounter initialization took ${initTime.toFixed(2)}ms`);
   }
 
   public countTokens(content: string, filePath?: string): number {

src/core/metrics/tokenCounterFactory.ts

@@ -0,0 +1,29 @@
+import type { TiktokenEncoding } from 'tiktoken';
+import { TokenCounter } from './TokenCounter.js';
+
+// Worker-level cache for TokenCounter instances by encoding
+const tokenCounters = new Map<TiktokenEncoding, TokenCounter>();
+
+/**
+ * Get or create a TokenCounter instance for the given encoding.
+ * This ensures only one TokenCounter exists per encoding per worker thread to optimize memory usage.
+ */
+export const getTokenCounter = (encoding: TiktokenEncoding): TokenCounter => {
+  let tokenCounter = tokenCounters.get(encoding);
+  if (!tokenCounter) {
+    tokenCounter = new TokenCounter(encoding);
+    tokenCounters.set(encoding, tokenCounter);
+  }
+  return tokenCounter;
+};
+
+/**
+ * Free all TokenCounter resources and clear the cache.
+ * This should be called when the worker is terminating.
+ */
+export const freeTokenCounter = (): void => {
+  for (const tokenCounter of tokenCounters.values()) {
+    tokenCounter.free();
+  }
+  tokenCounters.clear();
+};

src/core/metrics/workers/fileMetricsWorker.ts

@@ -1,29 +1,20 @@
 import type { TiktokenEncoding } from 'tiktoken';
-import { logger, setLogLevelByEnv } from '../../../shared/logger.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
 import type { ProcessedFile } from '../../file/fileTypes.js';
-import { TokenCounter } from '../TokenCounter.js';
+import { freeTokenCounter, getTokenCounter } from '../tokenCounterFactory.js';
 import type { FileMetrics } from './types.js';
 
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
+
 export interface FileMetricsTask {
   file: ProcessedFile;
   index: number;
   totalFiles: number;
   encoding: TiktokenEncoding;
 }
 
-// Worker-level singleton for TokenCounter
-let tokenCounter: TokenCounter | null = null;
-
-const getTokenCounter = (encoding: TiktokenEncoding): TokenCounter => {
-  if (!tokenCounter) {
-    tokenCounter = new TokenCounter(encoding);
-  }
-  return tokenCounter;
-};
-
-// Set logger log level from environment variable if provided
-setLogLevelByEnv();
-
 export default async ({ file, encoding }: FileMetricsTask): Promise<FileMetrics> => {
   const processStartAt = process.hrtime.bigint();
   const metrics = await calculateIndividualFileMetrics(file, encoding);
@@ -48,8 +39,5 @@ export const calculateIndividualFileMetrics = async (
 
 // Cleanup when worker is terminated
 process.on('exit', () => {
-  if (tokenCounter) {
-    tokenCounter.free();
-    tokenCounter = null;
-  }
+  freeTokenCounter();
 });

src/core/metrics/workers/outputMetricsWorker.ts

@@ -1,26 +1,17 @@
 import type { TiktokenEncoding } from 'tiktoken';
-import { logger, setLogLevelByEnv } from '../../../shared/logger.js';
-import { TokenCounter } from '../TokenCounter.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
+import { freeTokenCounter, getTokenCounter } from '../tokenCounterFactory.js';
+
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
 
 export interface OutputMetricsTask {
   content: string;
   encoding: TiktokenEncoding;
   path?: string;
 }
 
-// Worker-level singleton for TokenCounter
-let tokenCounter: TokenCounter | null = null;
-
-const getTokenCounter = (encoding: TiktokenEncoding): TokenCounter => {
-  if (!tokenCounter) {
-    tokenCounter = new TokenCounter(encoding);
-  }
-  return tokenCounter;
-};
-
-// Set logger log level from environment variable if provided
-setLogLevelByEnv();
-
 export default async ({ content, encoding, path }: OutputMetricsTask): Promise<number> => {
   const processStartAt = process.hrtime.bigint();
   const counter = getTokenCounter(encoding);
@@ -36,8 +27,5 @@ export default async ({ content, encoding, path }: OutputMetricsTask): Promise<n
 
 // Cleanup when worker is terminated
 process.on('exit', () => {
-  if (tokenCounter) {
-    tokenCounter.free();
-    tokenCounter = null;
-  }
+  freeTokenCounter();
 });

src/core/output/outputGenerate.ts

@@ -94,11 +94,16 @@ const generateParsableXmlOutput = async (renderContext: RenderContext): Promise<
   } catch (error) {
     throw new RepomixError(
       `Failed to generate XML output: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      error instanceof Error ? { cause: error } : undefined,
     );
   }
 };
 
-const generateHandlebarOutput = async (config: RepomixConfigMerged, renderContext: RenderContext): Promise<string> => {
+const generateHandlebarOutput = async (
+  config: RepomixConfigMerged,
+  renderContext: RenderContext,
+  processedFiles?: ProcessedFile[],
+): Promise<string> => {
   let template: string;
   switch (config.output.style) {
     case 'xml':
@@ -118,7 +123,30 @@ const generateHandlebarOutput = async (config: RepomixConfigMerged, renderContex
     const compiledTemplate = Handlebars.compile(template);
     return `${compiledTemplate(renderContext).trim()}\n`;
   } catch (error) {
-    throw new RepomixError(`Failed to compile template: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    if (error instanceof RangeError && error.message === 'Invalid string length') {
+      let largeFilesInfo = '';
+      if (processedFiles && processedFiles.length > 0) {
+        const topFiles = processedFiles
+          .sort((a, b) => b.content.length - a.content.length)
+          .slice(0, 5)
+          .map((f) => `  - ${f.path} (${(f.content.length / 1024 / 1024).toFixed(1)} MB)`)
+          .join('\n');
+        largeFilesInfo = `\n\nLargest files in this repository:\n${topFiles}`;
+      }
+
+      throw new RepomixError(
+        `Output size exceeds JavaScript string limit. The repository contains files that are too large to process.
+Please try:
+  - Use --ignore to exclude large files (e.g., --ignore "docs/**" or --ignore "*.html")
+  - Use --include to process only specific files
+  - Process smaller portions of the repository at a time${largeFilesInfo}`,
+        { cause: error },
+      );
+    }
+    throw new RepomixError(
+      `Failed to compile template: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      error instanceof Error ? { cause: error } : undefined,
+    );
   }
 };
 
@@ -147,14 +175,14 @@ export const generateOutput = async (
   );
   const renderContext = createRenderContext(outputGeneratorContext);
 
-  if (!config.output.parsableStyle) return deps.generateHandlebarOutput(config, renderContext);
+  if (!config.output.parsableStyle) return deps.generateHandlebarOutput(config, renderContext, sortedProcessedFiles);
   switch (config.output.style) {
     case 'xml':
       return deps.generateParsableXmlOutput(renderContext);
     case 'markdown':
-      return deps.generateHandlebarOutput(config, renderContext);
+      return deps.generateHandlebarOutput(config, renderContext, sortedProcessedFiles);
     default:
-      return deps.generateHandlebarOutput(config, renderContext);
+      return deps.generateHandlebarOutput(config, renderContext, sortedProcessedFiles);
   }
 };
 

src/core/packager.ts

@@ -1,4 +1,5 @@
 import type { RepomixConfigMerged } from '../config/configSchema.js';
+import { RepomixError } from '../shared/errorHandle.js';
 import type { RepomixProgressCallback } from '../shared/types.js';
 import { collectFiles } from './file/fileCollect.js';
 import { sortPaths } from './file/filePathSort.js';
@@ -32,7 +33,7 @@ const defaultDeps = {
   processFiles,
   generateOutput,
   validateFileSafety,
-  handleOutput: writeOutputToDisk,
+  writeOutputToDisk,
   copyToClipboardIfEnabled,
   calculateMetrics,
   sortPaths,
@@ -97,7 +98,7 @@ export const pack = async (
   const output = await deps.generateOutput(rootDirs, config, processedFiles, safeFilePaths, gitDiffResult);
 
   progressCallback('Writing output file...');
-  await deps.handleOutput(output, config);
+  await deps.writeOutputToDisk(output, config);
 
   await deps.copyToClipboardIfEnabled(output, progressCallback, config);
 

src/core/security/workers/securityCheckWorker.ts

@@ -1,7 +1,11 @@
 import { lintSource } from '@secretlint/core';
 import { creator } from '@secretlint/secretlint-rule-preset-recommend';
 import type { SecretLintCoreConfig } from '@secretlint/types';
-import { logger, setLogLevelByEnv } from '../../../shared/logger.js';
+import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
+
+// Initialize logger configuration from workerData at module load time
+// This must be called before any logging operations in the worker
+setLogLevelByWorkerData();
 
 // Security check type to distinguish between regular files and git diffs
 export type SecurityCheckType = 'file' | 'gitDiff';
@@ -18,9 +22,6 @@ export interface SuspiciousFileResult {
   type: SecurityCheckType;
 }
 
-// Set logger log level from environment variable if provided
-setLogLevelByEnv();
-
 export default async ({ filePath, content, type }: SecurityCheckTask) => {
   const config = createSecretLintConfig();
 

src/shared/errorHandle.ts

@@ -3,15 +3,15 @@ import { REPOMIX_DISCORD_URL, REPOMIX_ISSUES_URL } from './constants.js';
 import { logger, repomixLogLevels } from './logger.js';
 
 export class RepomixError extends Error {
-  constructor(message: string) {
-    super(message);
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
     this.name = 'RepomixError';
   }
 }
 
 export class RepomixConfigValidationError extends RepomixError {
-  constructor(message: string) {
-    super(message);
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
     this.name = 'RepomixConfigValidationError';
   }
 }
@@ -21,8 +21,16 @@ export const handleError = (error: unknown): void => {
 
   if (error instanceof RepomixError) {
     logger.error(`✖ ${error.message}`);
+    if (logger.getLogLevel() < repomixLogLevels.DEBUG) {
+      logger.log('');
+      logger.note('For detailed debug information, use the --verbose flag');
+    }
     // If expected error, show stack trace for debugging
     logger.debug('Stack trace:', error.stack);
+    // Show cause if available
+    if (error.cause) {
+      logger.debug('Caused by:', error.cause);
+    }
   } else if (error instanceof Error) {
     logger.error(`✖ Unexpected error: ${error.message}`);
     // If unexpected error, show stack trace by default
@@ -37,6 +45,7 @@ export const handleError = (error: unknown): void => {
     logger.error('✖ An unknown error occurred');
 
     if (logger.getLogLevel() < repomixLogLevels.DEBUG) {
+      logger.log('');
       logger.note('For detailed debug information, use the --verbose flag');
     }
   }

src/shared/logger.ts

@@ -1,4 +1,5 @@
 import util from 'node:util';
+import { workerData } from 'node:worker_threads';
 import pc from 'picocolors';
 
 export const repomixLogLevels = {
@@ -92,18 +93,20 @@ export const setLogLevel = (level: RepomixLogLevel) => {
 };
 
 /**
- * Set logger log level from REPOMIX_LOGLEVEL environment variable if valid.
+ * Set logger log level from workerData if valid.
+ * This is used in worker threads where configuration is passed via workerData.
  */
-export const setLogLevelByEnv = () => {
-  const logLevelStr = process.env.REPOMIX_LOGLEVEL;
-  const logLevelNum = Number(logLevelStr);
-  if (
-    logLevelNum === repomixLogLevels.SILENT ||
-    logLevelNum === repomixLogLevels.ERROR ||
-    logLevelNum === repomixLogLevels.WARN ||
-    logLevelNum === repomixLogLevels.INFO ||
-    logLevelNum === repomixLogLevels.DEBUG
-  ) {
-    setLogLevel(logLevelNum);
+export const setLogLevelByWorkerData = () => {
+  if (Array.isArray(workerData) && workerData.length > 1 && workerData[1]?.logLevel !== undefined) {
+    const logLevel = workerData[1].logLevel;
+    if (
+      logLevel === repomixLogLevels.SILENT ||
+      logLevel === repomixLogLevels.ERROR ||
+      logLevel === repomixLogLevels.WARN ||
+      logLevel === repomixLogLevels.INFO ||
+      logLevel === repomixLogLevels.DEBUG
+    ) {
+      setLogLevel(logLevel);
+    }
   }
 };