Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
97d4171
fix(core): Fix TokenCounter memory leak in worker threads
yamadashy Jul 24, 2025
748ce7c
refactor(shared): Consolidate initTaskRunner implementations
yamadashy Jul 24, 2025
2400a26
fix(core): Remove unused import in fileMetricsWorker
yamadashy Jul 24, 2025
d191835
fix(core): Improve Bun runtime compatibility for worker pool cleanup
yamadashy Jul 24, 2025
aad3d81
improve(core): Enhance logging in TokenCounter cleanup
yamadashy Jul 24, 2025
7c0ac16
feat(website): Add memory usage logging for web server
yamadashy Jul 24, 2025
23bdb2e
refactor(benchmarks): Move memory tests to isolated benchmarks directory
yamadashy Jul 27, 2025
4121462
refactor(benchmarks): Remove redundant monitor script
yamadashy Jul 27, 2025
1b24e6f
refactor(benchmarks): Rename scripts to better reflect their purpose
yamadashy Jul 27, 2025
948a2f2
refactor(benchmarks): Convert memory tests to TypeScript
yamadashy Jul 27, 2025
9860269
feat(benchmarks): Simplify memory test script names
yamadashy Jul 27, 2025
34a9d0b
feat(benchmarks): Add memory:check script for quick testing
yamadashy Jul 27, 2025
2820a1b
feat(core): Add detailed memory usage logging with trace level
yamadashy Jul 27, 2025
67c06a1
fix(memory): Address PR review feedback and improve memory utilities
yamadashy Jul 27, 2025
cc8865d
style(benchmarks): Fix lint issues in memory test files
yamadashy Jul 27, 2025
6e58ad8
improve(core): Optimize memory usage with child_process runtime and b…
yamadashy Jul 27, 2025
7cfe658
feat(metrics): Create dedicated git diff worker for improved memory e…
yamadashy Jul 27, 2025
aa4eeb2
feat(scripts): Add memory monitoring commands for development
yamadashy Jul 27, 2025
dac1210
feat(core): Isolate globby usage in workers to prevent memory leaks
yamadashy Jul 27, 2025
de1e0bf
docs(benchmarks): Simplify memory benchmark README
yamadashy Jul 27, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions benchmarks/memory/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Dependencies
node_modules/

# Build output
dist/

# Test outputs
memory-test-output.txt
test-output-*.txt

# Test results
memory-history-*.json
memory-test-results-*.json

# npm
package-lock.json
npm-debug.log*

# TypeScript
*.tsbuildinfo

# OS files
.DS_Store
Thumbs.db
33 changes: 33 additions & 0 deletions benchmarks/memory/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Memory Benchmarks

Memory usage monitoring tools for repomix.

## Setup

```bash
cd benchmarks/memory
npm install
```

## Quick Start

```bash
# Quick memory leak check
npm run leak:quick

# Detailed analysis
npm run leak:analyze
```

## Available Scripts

- `npm run leak:quick` - Fast leak detection (20 iterations)
- `npm run leak:watch` - Continuous monitoring
- `npm run leak:analyze` - Comprehensive analysis with reports

## Understanding Results

- **Heap Memory**: JavaScript objects (should stabilize)
- **RSS Memory**: Total process memory (watch for growth > 100%)

Look for consistent upward trends that indicate memory leaks.
32 changes: 32 additions & 0 deletions benchmarks/memory/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{
"name": "@repomix/memory-benchmarks",
"version": "1.0.0",
"private": true,
"type": "module",
"description": "Memory usage benchmarks and leak detection for repomix",
"scripts": {
"build": "tsc",
"build:repomix": "cd ../.. && node --run build",
"build:all": "node --run build:repomix && node --run build",
"clean": "rm -rf dist",
"memory:check": "node --run build:all && node --expose-gc dist/simple-memory-test.js",
"memory:watch": "node --run build:all && node --expose-gc dist/simple-memory-test.js continuous",
"memory:analyze": "node --run build:all && node --expose-gc dist/memory-leak-test.js 200 500"
},
"dependencies": {
"repomix": "file:../.."
},
"devDependencies": {
"@types/node": "^20.0.0",
"typescript": "^5.3.0"
},
"engines": {
"node": ">=18.0.0"
},
"keywords": [
"benchmark",
"memory",
"performance",
"leak-detection"
]
}
285 changes: 285 additions & 0 deletions benchmarks/memory/src/memory-leak-test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,285 @@
#!/usr/bin/env node

/**
* Comprehensive memory leak test for runCli
* Tests multiple configurations and generates detailed reports
*/

import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { runCli } from 'repomix';
import type { MemoryHistory, MemoryTestSummary, MemoryUsage, TestConfig } from './types.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const projectRoot = path.resolve(__dirname, '..');

// Configuration
const DEFAULT_ITERATIONS = 500;
const DEFAULT_DELAY = 100;
const MEMORY_LOG_INTERVAL = 10;
const FORCE_GC_INTERVAL = 20;

// Test configurations
const TEST_CONFIGS: TestConfig[] = [
{
name: 'Local Directory (src/)',
args: ['.'],
cwd: projectRoot,
options: {
include: 'src/**/*.ts',
output: path.join(__dirname, '../test-output-1.txt'),
style: 'plain',
quiet: true,
},
},
{
name: 'Local Directory with compression',
args: ['.'],
cwd: projectRoot,
options: {
include: 'src/**/*.ts',
output: path.join(__dirname, '../test-output-2.txt'),
style: 'xml',
compress: true,
quiet: true,
},
},
{
name: 'Complex patterns',
args: ['.'],
cwd: projectRoot,
options: {
include: 'src/**/*.{ts,js}',
ignore: '**/*.test.ts,**/*.d.ts',
output: path.join(__dirname, '../test-output-3.txt'),
style: 'markdown',
quiet: true,
},
},
];

// Memory tracking
const memoryHistory: MemoryHistory[] = [];

const iterations = Number.parseInt(process.argv[2]) || DEFAULT_ITERATIONS;
const delay = Number.parseInt(process.argv[3]) || DEFAULT_DELAY;

console.log('🧪 Comprehensive Memory Leak Test');
console.log(`📋 Configuration: ${iterations} iterations, ${delay}ms delay`);
console.log(`🎯 Test Configurations: ${TEST_CONFIGS.length} different configs`);
console.log('🛑 Press Ctrl+C to stop\n');

function getMemoryUsage(): MemoryUsage {
const usage = process.memoryUsage();
const heapUsed = Math.round((usage.heapUsed / 1024 / 1024) * 100) / 100;
const heapTotal = Math.round((usage.heapTotal / 1024 / 1024) * 100) / 100;
const external = Math.round((usage.external / 1024 / 1024) * 100) / 100;
const rss = Math.round((usage.rss / 1024 / 1024) * 100) / 100;
const heapUsagePercent = Math.round((usage.heapUsed / usage.heapTotal) * 100 * 100) / 100;

return {
heapUsed,
heapTotal,
external,
rss,
heapUsagePercent,
};
}

function forceGC(): void {
if (global.gc) {
global.gc();
console.log('🗑️ Forced garbage collection');
}
}

function logMemoryUsage(iteration: number, configName: string, error: Error | null = null): void {
const usage = getMemoryUsage();
const timestamp = new Date().toISOString();

memoryHistory.push({
iteration,
configName,
timestamp,
...usage,
error: !!error,
});

const statusIcon = error ? '❌' : '✅';
const errorText = error ? ` (ERROR: ${error.message})` : '';

console.log(
`${statusIcon} Iteration ${iteration}: ${configName} - ` +
`Heap: ${usage.heapUsed}MB/${usage.heapTotal}MB (${usage.heapUsagePercent}%), ` +
`RSS: ${usage.rss}MB${errorText}`,
);
}

async function cleanupFiles(): Promise<void> {
const filesToClean = TEST_CONFIGS.map((config) => config.options.output);

for (const file of filesToClean) {
try {
await fs.unlink(file);
} catch (error) {
if (error instanceof Error && 'code' in error && error.code !== 'ENOENT') {
console.warn(`Failed to cleanup ${file}:`, error.message);
}
}
}
}

function analyzeMemoryTrends(): void {
if (memoryHistory.length < 10) return;

const recent = memoryHistory.slice(-10);
const initial = memoryHistory.slice(0, 10);

const avgRecentHeap = recent.reduce((sum, entry) => sum + entry.heapUsed, 0) / recent.length;
const avgInitialHeap = initial.reduce((sum, entry) => sum + entry.heapUsed, 0) / initial.length;
const avgRecentRSS = recent.reduce((sum, entry) => sum + entry.rss, 0) / recent.length;
const avgInitialRSS = initial.reduce((sum, entry) => sum + entry.rss, 0) / initial.length;

const heapGrowth = ((avgRecentHeap - avgInitialHeap) / avgInitialHeap) * 100;
const rssGrowth = ((avgRecentRSS - avgInitialRSS) / avgInitialRSS) * 100;

console.log('\n📊 Memory Trend Analysis:');
console.log(
` Heap Growth: ${heapGrowth.toFixed(2)}% (${avgInitialHeap.toFixed(2)}MB → ${avgRecentHeap.toFixed(2)}MB)`,
);
console.log(` RSS Growth: ${rssGrowth.toFixed(2)}% (${avgInitialRSS.toFixed(2)}MB → ${avgRecentRSS.toFixed(2)}MB)`);

if (heapGrowth > 50 || rssGrowth > 50) {
console.log('⚠️ WARNING: Significant memory growth detected - possible memory leak!');
}
}

async function saveMemoryHistory(): Promise<void> {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = path.join(__dirname, '..', `memory-test-results-${timestamp}.json`);

const summary: MemoryTestSummary = {
testInfo: {
iterations: memoryHistory.length,
configurations: TEST_CONFIGS.length,
startTime: memoryHistory[0]?.timestamp || '',
endTime: memoryHistory[memoryHistory.length - 1]?.timestamp || '',
},
memoryHistory,
analysis: {
peakHeapUsage: Math.max(...memoryHistory.map((h) => h.heapUsed)),
peakRSSUsage: Math.max(...memoryHistory.map((h) => h.rss)),
errorCount: memoryHistory.filter((h) => h.error).length,
averageHeapUsage: memoryHistory.reduce((sum, h) => sum + h.heapUsed, 0) / memoryHistory.length,
averageRSSUsage: memoryHistory.reduce((sum, h) => sum + h.rss, 0) / memoryHistory.length,
},
};

try {
await fs.writeFile(filename, JSON.stringify(summary, null, 2));
console.log(`\n💾 Memory test results saved to: ${filename}`);
} catch (error) {
console.error('Failed to save memory history:', error instanceof Error ? error.message : String(error));
}
}

async function runMemoryLeakTest(): Promise<void> {
// Log initial memory usage
console.log('📊 Initial Memory Usage:');
logMemoryUsage(0, 'Initial', null);

console.log('\n🚀 Starting test iterations...\n');

for (let i = 1; i <= iterations; i++) {
const config = TEST_CONFIGS[(i - 1) % TEST_CONFIGS.length];
let error: Error | null = null;

try {
// Run the CLI with current configuration
await runCli(config.args, config.cwd, config.options);

// Clean up output files after each run
await cleanupFiles();
} catch (err) {
error = err instanceof Error ? err : new Error(String(err));
}

// Log memory usage at specified intervals or on error
if (i % MEMORY_LOG_INTERVAL === 0 || error) {
logMemoryUsage(i, config.name, error);
}

// Force garbage collection at specified intervals
if (i % FORCE_GC_INTERVAL === 0) {
forceGC();
}

// Analyze trends periodically
if (i % (MEMORY_LOG_INTERVAL * 2) === 0 && i > 20) {
analyzeMemoryTrends();
}

// Add delay between iterations
if (delay > 0) {
await new Promise((resolve) => setTimeout(resolve, delay));
}
}

console.log('\n✅ Memory leak test completed!');

// Final analysis
console.log('\n📊 Final Memory Analysis:');
const finalUsage = getMemoryUsage();
const initialUsage = memoryHistory[0];

if (initialUsage) {
console.log(`Initial: Heap ${initialUsage.heapUsed}MB, RSS ${initialUsage.rss}MB`);
console.log(`Final: Heap ${finalUsage.heapUsed}MB, RSS ${finalUsage.rss}MB`);
console.log(
`Growth: Heap ${(((finalUsage.heapUsed - initialUsage.heapUsed) / initialUsage.heapUsed) * 100).toFixed(2)}%, RSS ${(((finalUsage.rss - initialUsage.rss) / initialUsage.rss) * 100).toFixed(2)}%`,
);
}

// Save results
await saveMemoryHistory();

// Final cleanup
await cleanupFiles();

console.log('\n🎉 Test completed successfully!');
}

// Handle process termination
process.on('SIGINT', async () => {
console.log('\n\n⚠️ Test interrupted by user');
await saveMemoryHistory();
await cleanupFiles();
process.exit(0);
});

process.on('uncaughtException', async (error) => {
console.error('\n❌ Uncaught exception:', error);
await saveMemoryHistory();
await cleanupFiles();
process.exit(1);
});

// Validate arguments
if (Number.isNaN(iterations) || iterations <= 0) {
console.error('❌ Invalid iterations count. Must be a positive number.');
process.exit(1);
}

if (Number.isNaN(delay) || delay < 0) {
console.error('❌ Invalid delay. Must be a non-negative number.');
process.exit(1);
}

// Run the test
runMemoryLeakTest().catch(async (error) => {
console.error('\n❌ Test failed:', error);
await saveMemoryHistory();
await cleanupFiles();
process.exit(1);
});
Loading
Loading