refactor(metrics): Replace tiktoken with gpt-tokenizer

package.json

@@ -86,6 +86,7 @@
     "fast-xml-parser": "^5.4.1",
     "git-url-parse": "^16.1.0",
     "globby": "^16.1.1",
+    "gpt-tokenizer": "^3.4.0",
     "handlebars": "^4.7.8",
     "iconv-lite": "^0.7.0",
     "is-binary-path": "^3.0.0",
@@ -97,7 +98,6 @@
     "minimatch": "^10.2.4",
     "picocolors": "^1.1.1",
     "tar": "^7.5.9",
-    "tiktoken": "^1.0.22",
     "tinypool": "^2.1.0",
     "web-tree-sitter": "^0.26.6",
     "zod": "^4.3.6"

src/config/configSchema.ts

@@ -1,5 +1,5 @@
-import type { TiktokenEncoding } from 'tiktoken';
 import { z } from 'zod';
+import { tokenEncodings } from '../core/metrics/tokenEncoding.js';
 
 // Output style enum
 export const repomixOutputStyleSchema = z.enum(['xml', 'markdown', 'json', 'plain']);
@@ -122,10 +122,7 @@ export const repomixConfigDefaultSchema = z.object({
     enableSecurityCheck: z.boolean().default(true),
   }),
   tokenCount: z.object({
-    encoding: z
-      .string()
-      .default('o200k_base')
-      .transform((val) => val as TiktokenEncoding),
+    encoding: z.enum(tokenEncodings).default('o200k_base'),
   }),
 });
 

src/core/metrics/TokenCounter.ts

@@ -1,19 +1,32 @@
-import { get_encoding, type Tiktoken, type TiktokenEncoding } from 'tiktoken';
+import { GptEncoding } from 'gpt-tokenizer/GptEncoding';
+import { resolveEncodingAsync } from 'gpt-tokenizer/resolveEncodingAsync';
 import { logger } from '../../shared/logger.js';
+import type { TokenEncoding } from './tokenEncoding.js';
 
 export class TokenCounter {
-  private encoding: Tiktoken;
+  private encoding: GptEncoding;
 
-  constructor(encodingName: TiktokenEncoding) {
+  private constructor(encoding: GptEncoding) {
+    this.encoding = encoding;
+  }
+
+  /**
+   * Create a TokenCounter instance asynchronously.
+   * Uses dynamic import to load only the required BPE encoding data,
+   * avoiding the cost of loading all encodings (~4MB) on every worker.
+   */
+  public static async create(encodingName: TokenEncoding): Promise<TokenCounter> {
     const startTime = process.hrtime.bigint();
 
-    // Setup encoding with the specified model
-    this.encoding = get_encoding(encodingName);
+    const ranks = await resolveEncodingAsync(encodingName);
+    const encoding = GptEncoding.getEncodingApi(encodingName, () => ranks);
 
     const endTime = process.hrtime.bigint();
     const initTime = Number(endTime - startTime) / 1e6; // Convert to milliseconds
 
     logger.debug(`TokenCounter initialization took ${initTime.toFixed(2)}ms`);
+
+    return new TokenCounter(encoding);
   }
 
   public countTokens(content: string, filePath?: string): number {
@@ -23,7 +36,7 @@ export class TokenCounter {
       // This treats special tokens as ordinary text rather than control tokens,
       // which is appropriate for general code/text analysis where we're not
       // actually sending the content to an LLM API.
-      return this.encoding.encode(content, [], []).length;
+      return this.encoding.encode(content, { disallowedSpecial: new Set() }).length;
     } catch (error) {
       let message = '';
       if (error instanceof Error) {
@@ -42,7 +55,8 @@ export class TokenCounter {
     }
   }
 
-  public free(): void {
-    this.encoding.free();
-  }
+  // No-op retained for public API backward compatibility.
+  // gpt-tokenizer is pure JavaScript — memory is managed by GC,
+  // unlike tiktoken which required explicit WASM resource cleanup.
+  public free(): void {}
 }

src/core/metrics/calculateOutputMetrics.ts

@@ -1,14 +1,14 @@
-import type { TiktokenEncoding } from 'tiktoken';
 import { logger } from '../../shared/logger.js';
 import type { TaskRunner } from '../../shared/processConcurrency.js';
+import type { TokenEncoding } from './tokenEncoding.js';
 import type { TokenCountTask } from './workers/calculateMetricsWorker.js';
 
 const CHUNK_SIZE = 1000;
 const MIN_CONTENT_LENGTH_FOR_PARALLEL = 1_000_000; // 1000KB
 
 export const calculateOutputMetrics = async (
   content: string,
-  encoding: TiktokenEncoding,
+  encoding: TokenEncoding,
   path: string | undefined,
   deps: { taskRunner: TaskRunner<TokenCountTask, number> },
 ): Promise<number> => {

src/core/metrics/calculateSelectiveFileMetrics.ts

@@ -1,16 +1,16 @@
 import pc from 'picocolors';
-import type { TiktokenEncoding } from 'tiktoken';
 import { logger } from '../../shared/logger.js';
 import type { TaskRunner } from '../../shared/processConcurrency.js';
 import type { RepomixProgressCallback } from '../../shared/types.js';
 import type { ProcessedFile } from '../file/fileTypes.js';
+import type { TokenEncoding } from './tokenEncoding.js';
 import type { TokenCountTask } from './workers/calculateMetricsWorker.js';
 import type { FileMetrics } from './workers/types.js';
 
 export const calculateSelectiveFileMetrics = async (
   processedFiles: ProcessedFile[],
   targetFilePaths: string[],
-  tokenCounterEncoding: TiktokenEncoding,
+  tokenCounterEncoding: TokenEncoding,
   progressCallback: RepomixProgressCallback,
   deps: { taskRunner: TaskRunner<TokenCountTask, number> },
 ): Promise<FileMetrics[]> => {

src/core/metrics/tokenCounterFactory.ts

@@ -1,31 +1,33 @@
-import type { TiktokenEncoding } from 'tiktoken';
 import { logger } from '../../shared/logger.js';
 import { TokenCounter } from './TokenCounter.js';
+import type { TokenEncoding } from './tokenEncoding.js';
 
 // Worker-level cache for TokenCounter instances by encoding
-const tokenCounters = new Map<TiktokenEncoding, TokenCounter>();
+const tokenCounters = new Map<TokenEncoding, TokenCounter>();
 
 /**
  * Get or create a TokenCounter instance for the given encoding.
  * This ensures only one TokenCounter exists per encoding per worker thread to optimize memory usage.
  */
-export const getTokenCounter = (encoding: TiktokenEncoding): TokenCounter => {
+export const getTokenCounter = async (encoding: TokenEncoding): Promise<TokenCounter> => {
   let tokenCounter = tokenCounters.get(encoding);
   if (!tokenCounter) {
-    tokenCounter = new TokenCounter(encoding);
-    tokenCounters.set(encoding, tokenCounter);
+    tokenCounter = await TokenCounter.create(encoding);
+    // Guard against concurrent calls: only set if no other call populated the cache
+    if (!tokenCounters.has(encoding)) {
+      tokenCounters.set(encoding, tokenCounter);
+    } else {
+      tokenCounter = tokenCounters.get(encoding)!;
+    }
   }
   return tokenCounter;
 };
 
 /**
- * Free all TokenCounter resources and clear the cache.
+ * Clear all TokenCounter instances from the cache.
  * This should be called when the worker is terminating.
  */
 export const freeTokenCounters = (): void => {
-  for (const [encoding, tokenCounter] of tokenCounters.entries()) {
-    tokenCounter.free();
-    logger.debug(`Freed TokenCounter resources for encoding: ${encoding}`);
-  }
   tokenCounters.clear();
+  logger.debug('Cleared TokenCounter cache');
 };

src/core/metrics/tokenEncoding.ts

@@ -0,0 +1,14 @@
+/**
+ * Supported token encoding names.
+ * These match the encoding names supported by gpt-tokenizer.
+ */
+export const tokenEncodings = [
+  'o200k_base',
+  'o200k_harmony',
+  'cl100k_base',
+  'p50k_base',
+  'p50k_edit',
+  'r50k_base',
+] as const;
+
+export type TokenEncoding = (typeof tokenEncodings)[number];

src/core/metrics/workers/calculateMetricsWorker.ts

@@ -1,13 +1,13 @@
-import type { TiktokenEncoding } from 'tiktoken';
 import { logger, setLogLevelByWorkerData } from '../../../shared/logger.js';
 import { freeTokenCounters, getTokenCounter } from '../tokenCounterFactory.js';
+import type { TokenEncoding } from '../tokenEncoding.js';
 
 /**
  * Simple token counting worker for metrics calculation.
  *
- * This worker provides a focused interface for counting tokens from text content,
- * using the Tiktoken encoding. All complex metric calculation logic is handled
- * by the calling side to maintain separation of concerns.
+ * This worker provides a focused interface for counting tokens from text content.
+ * All complex metric calculation logic is handled by the calling side to maintain
+ * separation of concerns.
  */
 
 // Initialize logger configuration from workerData at module load time
@@ -16,15 +16,15 @@ setLogLevelByWorkerData();
 
 export interface TokenCountTask {
   content: string;
-  encoding: TiktokenEncoding;
+  encoding: TokenEncoding;
   path?: string;
 }
 
 export const countTokens = async (task: TokenCountTask): Promise<number> => {
   const processStartAt = process.hrtime.bigint();
 
   try {
-    const counter = getTokenCounter(task.encoding);
+    const counter = await getTokenCounter(task.encoding);
     const tokenCount = counter.countTokens(task.content, task.path);
 
     logger.trace(`Counted tokens. Count: ${tokenCount}. Took: ${getProcessDuration(processStartAt)}ms`);