diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 582507aaf..9f9f4c671 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 15 defaults: run: - working-directory: benchmarks/memory + working-directory: scripts/memory steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # ratchet:actions/setup-node@v4 @@ -32,26 +32,26 @@ jobs: cache: npm cache-dependency-path: | package-lock.json - benchmarks/memory/package-lock.json - + scripts/memory/package-lock.json + # Install root dependencies (for repomix) - name: Install root dependencies working-directory: . run: npm ci - + # Build repomix - name: Build repomix working-directory: . run: npm run build - + # Install benchmark dependencies - name: Install benchmark dependencies run: npm ci - + # Build benchmark - name: Build benchmark run: npm run build - + # Run memory test - name: Run memory test run: | @@ -60,14 +60,14 @@ jobs: node --expose-gc dist/memory-test.js $ITERATIONS $DELAY env: CI: true - + # Upload memory test results if available - name: Upload memory test results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4 if: always() with: name: memory-test-results-${{ github.run_id }} - path: benchmarks/memory/memory-test-results-*.json + path: scripts/memory/memory-test-results-*.json retention-days: 30 memory-test-full: @@ -78,7 +78,7 @@ jobs: if: github.event_name == 'workflow_dispatch' defaults: run: - working-directory: benchmarks/memory + working-directory: scripts/memory steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # ratchet:actions/setup-node@v4 @@ -87,37 +87,37 @@ jobs: cache: npm cache-dependency-path: | package-lock.json - benchmarks/memory/package-lock.json - + scripts/memory/package-lock.json + # Install root dependencies (for repomix) - name: Install root dependencies working-directory: . run: npm ci - + # Build repomix - name: Build repomix working-directory: . run: npm run build - + # Install benchmark dependencies - name: Install benchmark dependencies run: npm ci - + # Build benchmark - name: Build benchmark run: npm run build - + # Run comprehensive memory test - name: Run comprehensive memory test run: node --expose-gc dist/memory-test.js --full --save env: CI: true - + # Upload detailed memory test results - name: Upload detailed memory test results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4 if: always() with: name: memory-test-results-full-${{ github.run_id }} - path: benchmarks/memory/memory-test-results-*.json - retention-days: 90 \ No newline at end of file + path: scripts/memory/memory-test-results-*.json + retention-days: 90 diff --git a/benchmarks/memory/.gitignore b/scripts/memory/.gitignore similarity index 90% rename from benchmarks/memory/.gitignore rename to scripts/memory/.gitignore index a289d198f..7aa983fd5 100644 --- a/benchmarks/memory/.gitignore +++ b/scripts/memory/.gitignore @@ -7,6 +7,7 @@ dist/ # Test outputs memory-test-output.txt test-output-*.txt +test-output.txt # Test results memory-history-*.json @@ -20,4 +21,4 @@ npm-debug.log* # OS files .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db diff --git a/benchmarks/memory/README.md b/scripts/memory/README.md similarity index 87% rename from benchmarks/memory/README.md rename to scripts/memory/README.md index 96b6db2bf..11dbd83b2 100644 --- a/benchmarks/memory/README.md +++ b/scripts/memory/README.md @@ -5,7 +5,7 @@ Memory usage monitoring tools for repomix. ## Setup ```bash -cd benchmarks/memory +cd scripts/memory npm install ``` @@ -30,4 +30,4 @@ npm run leak:analyze - **Heap Memory**: JavaScript objects (should stabilize) - **RSS Memory**: Total process memory (watch for growth > 100%) -Look for consistent upward trends that indicate memory leaks. \ No newline at end of file +Look for consistent upward trends that indicate memory leaks. diff --git a/benchmarks/memory/package-lock.json b/scripts/memory/package-lock.json similarity index 82% rename from benchmarks/memory/package-lock.json rename to scripts/memory/package-lock.json index f4a703c6b..21643ce91 100644 --- a/benchmarks/memory/package-lock.json +++ b/scripts/memory/package-lock.json @@ -8,6 +8,8 @@ "name": "@repomix/memory-benchmarks", "version": "1.0.0", "dependencies": { + "@types/asciichart": "^1.5.8", + "asciichart": "^1.5.25", "repomix": "file:../.." }, "devDependencies": { @@ -19,7 +21,7 @@ } }, "../..": { - "version": "1.4.0", + "version": "1.4.1", "license": "MIT", "dependencies": { "@clack/prompts": "^0.10.1", @@ -71,6 +73,12 @@ "yarn": ">=1.22.22" } }, + "node_modules/@types/asciichart": { + "version": "1.5.8", + "resolved": "https://registry.npmjs.org/@types/asciichart/-/asciichart-1.5.8.tgz", + "integrity": "sha512-8yzgCUybv8/yUfj4WeTh7G+V+AxU7AzwsF2CrkTtARKHdrxE/EiByF2efUMj6qdm87tENucNu6pLs22RWU0H7g==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.17.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.17.2.tgz", @@ -81,6 +89,12 @@ "undici-types": "~6.21.0" } }, + "node_modules/asciichart": { + "version": "1.5.25", + "resolved": "https://registry.npmjs.org/asciichart/-/asciichart-1.5.25.tgz", + "integrity": "sha512-PNxzXIPPOtWq8T7bgzBtk9cI2lgS4SJZthUHEiQ1aoIc3lNzGfUvIvo9LiAnq26TACo9t1/4qP6KTGAUbzX9Xg==", + "license": "MIT" + }, "node_modules/repomix": { "resolved": "../..", "link": true diff --git a/benchmarks/memory/package.json b/scripts/memory/package.json similarity index 93% rename from benchmarks/memory/package.json rename to scripts/memory/package.json index 7f9098da3..c892a390a 100644 --- a/benchmarks/memory/package.json +++ b/scripts/memory/package.json @@ -14,6 +14,8 @@ "test:continuous": "node --run build:all && node --expose-gc dist/memory-test.js --continuous" }, "dependencies": { + "@types/asciichart": "^1.5.8", + "asciichart": "^1.5.25", "repomix": "file:../.." }, "devDependencies": { diff --git a/benchmarks/memory/src/memory-test.ts b/scripts/memory/src/memory-test.ts similarity index 83% rename from benchmarks/memory/src/memory-test.ts rename to scripts/memory/src/memory-test.ts index fbb6a5c6c..9fd00bd77 100644 --- a/benchmarks/memory/src/memory-test.ts +++ b/scripts/memory/src/memory-test.ts @@ -9,6 +9,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import * as asciichart from 'asciichart'; import { runCli } from 'repomix'; import type { MemoryHistory, MemoryTestSummary, MemoryUsage, TestConfig } from './types.js'; @@ -29,11 +30,16 @@ const numericArgs = args.filter((arg) => !arg.startsWith('-') && !Number.isNaN(N const iterations = Number(numericArgs[0]) || (flags.full ? 200 : 100); const delay = Number(numericArgs[1]) || (flags.full ? 100 : 50); -// Configuration +// Configuration constants const MEMORY_LOG_INTERVAL = flags.full ? 10 : 5; -const FORCE_GC_INTERVAL = flags.full ? 20 : 10; +const FORCE_GC_INTERVAL = flags.full ? 50 : 20; const WARNING_THRESHOLD = flags.full ? 50 : 100; // Memory growth percentage +// Graph display constants +const MIN_POINTS_FOR_GRAPH = 5; +const GRAPH_DATA_POINTS = 40; +const GRAPH_HEIGHT = 8; + // Test configuration const TEST_CONFIG: TestConfig = { name: 'Memory Test', @@ -42,7 +48,7 @@ const TEST_CONFIG: TestConfig = { options: { include: 'src/**/*.ts', output: path.join(__dirname, '../test-output.txt'), - style: 'plain', + compress: true, quiet: true, }, }; @@ -92,9 +98,7 @@ function getMemoryUsage(): MemoryUsage { function forceGC(): void { if (global.gc) { global.gc(); - if (flags.full) { - console.log('๐Ÿ—‘๏ธ Forced garbage collection'); - } + console.log('๐Ÿ—‘๏ธ Forced garbage collection'); } } @@ -113,10 +117,18 @@ function logMemoryUsage(iteration: number, configName: string, error: Error | nu const statusIcon = error ? 'โŒ' : 'โœ…'; const errorText = error ? ` (ERROR: ${error.message})` : ''; + // Format with fixed widths for alignment + const iterationStr = `Iteration ${iteration.toString().padStart(3)}`; + const configStr = configName.padEnd(12); + const heapStr = `${usage.heapUsed.toString().padStart(6)}MB`; + const heapTotalStr = `${usage.heapTotal.toString().padStart(6)}MB`; + const heapPercentStr = `(${usage.heapUsagePercent.toString().padStart(5)}%)`; + const rssStr = `${usage.rss.toString().padStart(6)}MB`; + console.log( - `${statusIcon} Iteration ${iteration}: ${configName} - ` + - `Heap: ${usage.heapUsed}MB/${usage.heapTotal}MB (${usage.heapUsagePercent}%), ` + - `RSS: ${usage.rss}MB${errorText}`, + `${statusIcon} ${iterationStr}: ${configStr} - ` + + `Heap: ${heapStr}/${heapTotalStr} ${heapPercentStr}, ` + + `RSS: ${rssStr}${errorText}`, ); } @@ -130,6 +142,33 @@ async function cleanupFiles(): Promise { } } +function displayMemoryGraphs(history: MemoryHistory[]): void { + if (history.length < MIN_POINTS_FOR_GRAPH) return; + + const recentHistory = history.slice(-GRAPH_DATA_POINTS); + + const heapData = recentHistory.map((entry) => entry.heapUsed); + const rssData = recentHistory.map((entry) => entry.rss); + + console.log('\n๐Ÿ“ˆ Memory Usage Graphs:'); + + console.log('\n๐Ÿ”ธ Heap Usage (MB):'); + console.log( + asciichart.plot(heapData, { + height: GRAPH_HEIGHT, + format: (x: number) => x.toFixed(1), + }), + ); + + console.log('\n๐Ÿ”น RSS Usage (MB):'); + console.log( + asciichart.plot(rssData, { + height: GRAPH_HEIGHT, + format: (x: number) => x.toFixed(1), + }), + ); +} + function analyzeMemoryTrends(): void { if (memoryHistory.length < 10) return; @@ -153,6 +192,9 @@ function analyzeMemoryTrends(): void { if (heapGrowth > WARNING_THRESHOLD || rssGrowth > WARNING_THRESHOLD) { console.log('โš ๏ธ WARNING: Significant memory growth detected - possible memory leak!'); } + + // Show graphs + displayMemoryGraphs(memoryHistory); } async function saveMemoryHistory(): Promise { @@ -263,6 +305,12 @@ async function runMemoryTest(): Promise { } } + // Show final graph + if (memoryHistory.length >= MIN_POINTS_FOR_GRAPH) { + console.log('\n๐Ÿ“ˆ Complete Memory Usage Timeline:'); + displayMemoryGraphs(memoryHistory); + } + // Save results if requested await saveMemoryHistory(); @@ -307,7 +355,12 @@ console.log('๐Ÿงช Memory Test'); console.log(`๐Ÿ“‹ Mode: ${flags.full ? 'Comprehensive' : 'Basic'} (${iterations} iterations, ${delay}ms delay)`); console.log( `โšก Features: ${ - [flags.continuous && 'Continuous Mode', flags.saveResults && 'Save Results', flags.full && 'Full Analysis'] + [ + flags.continuous && 'Continuous Mode', + flags.saveResults && 'Save Results', + flags.full && 'Full Analysis', + 'Graph Display', + ] .filter(Boolean) .join(', ') || 'Basic Test' }`, diff --git a/benchmarks/memory/src/types.ts b/scripts/memory/src/types.ts similarity index 95% rename from benchmarks/memory/src/types.ts rename to scripts/memory/src/types.ts index 242316916..9b514cde2 100644 --- a/benchmarks/memory/src/types.ts +++ b/scripts/memory/src/types.ts @@ -26,7 +26,6 @@ export interface TestConfig { include?: string; ignore?: string; output: string; - style: 'plain' | 'xml' | 'markdown'; compress?: boolean; quiet: boolean; }; diff --git a/benchmarks/memory/tsconfig.json b/scripts/memory/tsconfig.json similarity index 100% rename from benchmarks/memory/tsconfig.json rename to scripts/memory/tsconfig.json diff --git a/src/core/file/fileCollect.ts b/src/core/file/fileCollect.ts index e22668bf5..e5f1c2894 100644 --- a/src/core/file/fileCollect.ts +++ b/src/core/file/fileCollect.ts @@ -26,7 +26,6 @@ export const collectFiles = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: filePaths.length, workerPath: new URL('./workers/fileCollectWorker.js', import.meta.url).href, - // Use worker_threads for file collection - low memory leak risk runtime: 'worker_threads', }); const tasks = filePaths.map( diff --git a/src/core/file/fileProcess.ts b/src/core/file/fileProcess.ts index d113f9d96..5a1483412 100644 --- a/src/core/file/fileProcess.ts +++ b/src/core/file/fileProcess.ts @@ -24,6 +24,8 @@ export const processFiles = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: rawFiles.length, workerPath: new URL('./workers/fileProcessWorker.js', import.meta.url).href, + // High memory usage and leak risk + runtime: 'child_process', }); const tasks = rawFiles.map( (rawFile, _index) => diff --git a/src/core/file/globbyExecute.ts b/src/core/file/globbyExecute.ts index 1a424cd6d..1f7200078 100644 --- a/src/core/file/globbyExecute.ts +++ b/src/core/file/globbyExecute.ts @@ -16,6 +16,7 @@ export const executeGlobbyInWorker = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: 1, workerPath: new URL('./workers/globbyWorker.js', import.meta.url).href, + runtime: 'child_process', }); try { diff --git a/src/core/file/workers/fileCollectWorker.ts b/src/core/file/workers/fileCollectWorker.ts index 364329010..875ef337c 100644 --- a/src/core/file/workers/fileCollectWorker.ts +++ b/src/core/file/workers/fileCollectWorker.ts @@ -49,3 +49,8 @@ export default async ({ filePath, rootDir, maxFileSize }: FileCollectTask): Prom `File processing for ${filePath} resulted in an unexpected state: content is null but no skip reason was provided.`, ); }; + +// Export cleanup function for Tinypool teardown (no cleanup needed for this worker) +export const onWorkerTermination = () => { + // No cleanup needed for file collection worker +}; diff --git a/src/core/file/workers/fileProcessWorker.ts b/src/core/file/workers/fileProcessWorker.ts index 0435592de..3e51da52c 100644 --- a/src/core/file/workers/fileProcessWorker.ts +++ b/src/core/file/workers/fileProcessWorker.ts @@ -1,5 +1,6 @@ import type { RepomixConfigMerged } from '../../../config/configSchema.js'; import { setLogLevelByWorkerData } from '../../../shared/logger.js'; +import { cleanupLanguageParser } from '../../treeSitter/parseFile.js'; import { processContent } from '../fileProcessContent.js'; import type { ProcessedFile, RawFile } from '../fileTypes.js'; @@ -19,3 +20,8 @@ export default async ({ rawFile, config }: FileProcessTask): Promise { + await cleanupLanguageParser(); +}; diff --git a/src/core/file/workers/globbyWorker.ts b/src/core/file/workers/globbyWorker.ts index 682e07d3e..cbcb6b685 100644 --- a/src/core/file/workers/globbyWorker.ts +++ b/src/core/file/workers/globbyWorker.ts @@ -8,3 +8,8 @@ export interface GlobbyTask { export default async ({ patterns, options }: GlobbyTask): Promise => { return globby(patterns, options); }; + +// Export cleanup function for Tinypool teardown (no cleanup needed for this worker) +export const onWorkerTermination = () => { + // No cleanup needed for globby worker +}; diff --git a/src/core/metrics/calculateGitDiffMetrics.ts b/src/core/metrics/calculateGitDiffMetrics.ts index ff67aa2d2..7c2cadac5 100644 --- a/src/core/metrics/calculateGitDiffMetrics.ts +++ b/src/core/metrics/calculateGitDiffMetrics.ts @@ -26,6 +26,7 @@ export const calculateGitDiffMetrics = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: 1, // Single task for git diff calculation workerPath: new URL('./workers/gitDiffMetricsWorker.js', import.meta.url).href, + runtime: 'child_process', }); try { diff --git a/src/core/metrics/calculateGitLogMetrics.ts b/src/core/metrics/calculateGitLogMetrics.ts index 11e5bd253..f9ec1d6c7 100644 --- a/src/core/metrics/calculateGitLogMetrics.ts +++ b/src/core/metrics/calculateGitLogMetrics.ts @@ -31,6 +31,7 @@ export const calculateGitLogMetrics = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: 1, // Single task for git log calculation workerPath: new URL('./workers/gitLogMetricsWorker.js', import.meta.url).href, + runtime: 'child_process', }); try { diff --git a/src/core/metrics/calculateOutputMetrics.ts b/src/core/metrics/calculateOutputMetrics.ts index 16ae0bc9f..44d8200f0 100644 --- a/src/core/metrics/calculateOutputMetrics.ts +++ b/src/core/metrics/calculateOutputMetrics.ts @@ -19,6 +19,7 @@ export const calculateOutputMetrics = async ( const taskRunner = deps.initTaskRunner({ numOfTasks, workerPath: new URL('./workers/outputMetricsWorker.js', import.meta.url).href, + runtime: 'child_process', }); try { diff --git a/src/core/metrics/calculateSelectiveFileMetrics.ts b/src/core/metrics/calculateSelectiveFileMetrics.ts index 65f928cab..611520433 100644 --- a/src/core/metrics/calculateSelectiveFileMetrics.ts +++ b/src/core/metrics/calculateSelectiveFileMetrics.ts @@ -26,6 +26,7 @@ export const calculateSelectiveFileMetrics = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: filesToProcess.length, workerPath: new URL('./workers/fileMetricsWorker.js', import.meta.url).href, + runtime: 'child_process', }); const tasks = filesToProcess.map( (file, index) => diff --git a/src/core/metrics/workers/fileMetricsWorker.ts b/src/core/metrics/workers/fileMetricsWorker.ts index 399cb7c59..5f5bb86e5 100644 --- a/src/core/metrics/workers/fileMetricsWorker.ts +++ b/src/core/metrics/workers/fileMetricsWorker.ts @@ -37,7 +37,7 @@ export const calculateIndividualFileMetrics = async ( return { path: file.path, charCount, tokenCount }; }; -// Cleanup when worker is terminated -process.on('exit', () => { +// Export cleanup function for Tinypool teardown +export const onWorkerTermination = () => { freeTokenCounters(); -}); +}; diff --git a/src/core/metrics/workers/gitDiffMetricsWorker.ts b/src/core/metrics/workers/gitDiffMetricsWorker.ts index a285bc484..2aa97be30 100644 --- a/src/core/metrics/workers/gitDiffMetricsWorker.ts +++ b/src/core/metrics/workers/gitDiffMetricsWorker.ts @@ -36,7 +36,7 @@ export default async ({ workTreeDiffContent, stagedDiffContent, encoding }: GitD return totalTokens; }; -// Cleanup when worker is terminated -process.on('exit', () => { +// Export cleanup function for Tinypool teardown +export const onWorkerTermination = () => { freeTokenCounters(); -}); +}; diff --git a/src/core/metrics/workers/gitLogMetricsWorker.ts b/src/core/metrics/workers/gitLogMetricsWorker.ts index 338b9b8ee..aaa1f6f1e 100644 --- a/src/core/metrics/workers/gitLogMetricsWorker.ts +++ b/src/core/metrics/workers/gitLogMetricsWorker.ts @@ -30,8 +30,10 @@ export default async ({ content, encoding }: GitLogMetricsTask): Promise } catch (error) { logger.error('Error calculating git log token count:', error); return 0; - } finally { - // Clean up token counters to free memory - freeTokenCounters(); } }; + +// Export cleanup function for Tinypool teardown +export const onWorkerTermination = () => { + freeTokenCounters(); +}; diff --git a/src/core/metrics/workers/outputMetricsWorker.ts b/src/core/metrics/workers/outputMetricsWorker.ts index 738f6c68c..62ea93a56 100644 --- a/src/core/metrics/workers/outputMetricsWorker.ts +++ b/src/core/metrics/workers/outputMetricsWorker.ts @@ -25,7 +25,7 @@ export default async ({ content, encoding, path }: OutputMetricsTask): Promise { +// Export cleanup function for Tinypool teardown +export const onWorkerTermination = () => { freeTokenCounters(); -}); +}; diff --git a/src/core/security/securityCheck.ts b/src/core/security/securityCheck.ts index 47439c787..4dd1aa43b 100644 --- a/src/core/security/securityCheck.ts +++ b/src/core/security/securityCheck.ts @@ -58,6 +58,7 @@ export const runSecurityCheck = async ( const taskRunner = deps.initTaskRunner({ numOfTasks: rawFiles.length + gitDiffTasks.length + gitLogTasks.length, workerPath: new URL('./workers/securityCheckWorker.js', import.meta.url).href, + runtime: 'child_process', }); const fileTasks = rawFiles.map( (file) => diff --git a/src/core/security/workers/securityCheckWorker.ts b/src/core/security/workers/securityCheckWorker.ts index 2cd33244f..acc4c6b6d 100644 --- a/src/core/security/workers/securityCheckWorker.ts +++ b/src/core/security/workers/securityCheckWorker.ts @@ -83,3 +83,8 @@ export const createSecretLintConfig = (): SecretLintCoreConfig => ({ }, ], }); + +// Export cleanup function for Tinypool teardown (no cleanup needed for this worker) +export const onWorkerTermination = () => { + // No cleanup needed for security check worker +}; diff --git a/src/core/treeSitter/languageParser.ts b/src/core/treeSitter/languageParser.ts index d877af829..6d5888aec 100644 --- a/src/core/treeSitter/languageParser.ts +++ b/src/core/treeSitter/languageParser.ts @@ -2,12 +2,14 @@ import * as path from 'node:path'; import Parser from 'web-tree-sitter'; import { RepomixError } from '../../shared/errorHandle.js'; +import { logger } from '../../shared/logger.js'; import { ext2Lang } from './ext2Lang.js'; import { type SupportedLang, lang2Query } from './lang2Query.js'; import { loadLanguage } from './loadLanguage.js'; import { type ParseStrategy, createParseStrategy } from './parseStrategies/ParseStrategy.js'; interface LanguageResources { + lang: SupportedLang; parser: Parser; query: Parser.Query; strategy: ParseStrategy; @@ -30,6 +32,7 @@ export class LanguageParser { const strategy = createParseStrategy(name); const resources: LanguageResources = { + lang: name, parser, query, strategy, @@ -95,6 +98,7 @@ export class LanguageParser { public async dispose(): Promise { for (const resources of this.loadedResources.values()) { resources.parser.delete(); + logger.debug(`Deleted parser for language: ${resources.lang}`); } this.loadedResources.clear(); this.initialized = false; diff --git a/src/core/treeSitter/parseFile.ts b/src/core/treeSitter/parseFile.ts index 01947400b..03ec0c8e3 100644 --- a/src/core/treeSitter/parseFile.ts +++ b/src/core/treeSitter/parseFile.ts @@ -87,6 +87,21 @@ const getLanguageParserSingleton = async () => { } return languageParserSingleton; }; +/** + * Clean up the language parser singleton by deleting all loaded parsers + */ +export const cleanupLanguageParser = async (): Promise => { + if (languageParserSingleton) { + try { + await languageParserSingleton.dispose(); + logger.debug('Language parser singleton deleted'); + } catch (err) { + logger.debug('Language parser dispose threw', err); + } finally { + languageParserSingleton = null; + } + } +}; const filterDuplicatedChunks = (chunks: CapturedChunk[]): CapturedChunk[] => { // Group chunks by their start row diff --git a/src/shared/processConcurrency.ts b/src/shared/processConcurrency.ts index 3c84a269f..fde833a9e 100644 --- a/src/shared/processConcurrency.ts +++ b/src/shared/processConcurrency.ts @@ -7,7 +7,7 @@ export type WorkerRuntime = NonNullable; export interface WorkerOptions { numOfTasks: number; workerPath: string; - runtime?: WorkerRuntime; + runtime: WorkerRuntime; } // Worker initialization is expensive, so we prefer fewer threads unless there are many files @@ -47,6 +47,7 @@ export const createWorkerPool = (options: WorkerOptions): Tinypool => { minThreads, maxThreads, idleTimeout: 5000, + teardown: 'onWorkerTermination', workerData: { logLevel: logger.getLogLevel(), }, diff --git a/tests/shared/processConcurrency.test.ts b/tests/shared/processConcurrency.test.ts index d86fef201..2192d8d02 100644 --- a/tests/shared/processConcurrency.test.ts +++ b/tests/shared/processConcurrency.test.ts @@ -73,7 +73,7 @@ describe('processConcurrency', () => { it('should initialize Tinypool with correct configuration', () => { const workerPath = '/path/to/worker.js'; - const tinypool = createWorkerPool({ numOfTasks: 500, workerPath }); + const tinypool = createWorkerPool({ numOfTasks: 500, workerPath, runtime: 'child_process' }); expect(Tinypool).toHaveBeenCalledWith({ filename: workerPath, @@ -81,6 +81,7 @@ describe('processConcurrency', () => { minThreads: 1, maxThreads: 4, // Math.min(4, 500/100) = 4 idleTimeout: 5000, + teardown: 'onWorkerTermination', workerData: { logLevel: 2, }, @@ -98,6 +99,7 @@ describe('processConcurrency', () => { minThreads: 1, maxThreads: 4, // Math.min(4, 500/100) = 4 idleTimeout: 5000, + teardown: 'onWorkerTermination', workerData: { logLevel: 2, }, @@ -120,7 +122,7 @@ describe('processConcurrency', () => { it('should return a TaskRunner with run and cleanup methods', () => { const workerPath = '/path/to/worker.js'; - const taskRunner = initTaskRunner({ numOfTasks: 100, workerPath }); + const taskRunner = initTaskRunner({ numOfTasks: 100, workerPath, runtime: 'child_process' }); expect(taskRunner).toHaveProperty('run'); expect(taskRunner).toHaveProperty('cleanup'); diff --git a/website/client/src/public/schemas/1.4.2/schema.json b/website/client/src/public/schemas/1.4.2/schema.json new file mode 100644 index 000000000..4f6f2f9d0 --- /dev/null +++ b/website/client/src/public/schemas/1.4.2/schema.json @@ -0,0 +1,150 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "input": { + "type": "object", + "properties": { + "maxFileSize": { + "type": "number" + } + }, + "additionalProperties": false + }, + "output": { + "type": "object", + "properties": { + "filePath": { + "type": "string" + }, + "style": { + "type": "string", + "enum": [ + "xml", + "markdown", + "plain" + ] + }, + "parsableStyle": { + "type": "boolean" + }, + "headerText": { + "type": "string" + }, + "instructionFilePath": { + "type": "string" + }, + "fileSummary": { + "type": "boolean" + }, + "directoryStructure": { + "type": "boolean" + }, + "files": { + "type": "boolean" + }, + "removeComments": { + "type": "boolean" + }, + "removeEmptyLines": { + "type": "boolean" + }, + "compress": { + "type": "boolean" + }, + "topFilesLength": { + "type": "number" + }, + "showLineNumbers": { + "type": "boolean" + }, + "truncateBase64": { + "type": "boolean" + }, + "copyToClipboard": { + "type": "boolean" + }, + "includeEmptyDirectories": { + "type": "boolean" + }, + "tokenCountTree": { + "type": [ + "boolean", + "number", + "string" + ] + }, + "git": { + "type": "object", + "properties": { + "sortByChanges": { + "type": "boolean" + }, + "sortByChangesMaxCommits": { + "type": "number" + }, + "includeDiffs": { + "type": "boolean" + }, + "includeLogs": { + "type": "boolean" + }, + "includeLogsCount": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "include": { + "type": "array", + "items": { + "type": "string" + } + }, + "ignore": { + "type": "object", + "properties": { + "useGitignore": { + "type": "boolean" + }, + "useDefaultPatterns": { + "type": "boolean" + }, + "customPatterns": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "security": { + "type": "object", + "properties": { + "enableSecurityCheck": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "tokenCount": { + "type": "object", + "properties": { + "encoding": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "title": "Repomix Configuration", + "description": "Schema for repomix.config.json configuration file" +} \ No newline at end of file