Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/cli/actions/workers/defaultActionWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,14 @@ async function defaultActionWorker(

spinner.succeed('Packing completed successfully!');

// Strip file content from IPC response to reduce structured clone overhead.
// The main process only uses processedFiles[].path (for token count tree),
// not the content (~4MB savings for typical repos).
return {
packResult,
packResult: {
...packResult,
processedFiles: packResult.processedFiles.map((file) => ({ ...file, content: '' })),
},
config,
};
} catch (error) {
Expand Down
13 changes: 7 additions & 6 deletions src/cli/cliRun.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,10 @@ import process from 'node:process';
import { Option, program } from 'commander';
import pc from 'picocolors';
import { getVersion } from '../core/file/packageJsonParse.js';
import { isExplicitRemoteUrl } from '../core/git/gitRemoteParse.js';
import { handleError, RepomixError } from '../shared/errorHandle.js';
import { logger, repomixLogLevels } from '../shared/logger.js';
import { parseHumanSizeToBytes } from '../shared/sizeParse.js';
import { runDefaultAction } from './actions/defaultAction.js';
import { runInitAction } from './actions/initAction.js';
import { runMcpAction } from './actions/mcpAction.js';
import { runRemoteAction } from './actions/remoteAction.js';
import { runVersionAction } from './actions/versionAction.js';
import type { CliOptions } from './types.js';

// Semantic mapping for CLI suggestions
Expand Down Expand Up @@ -257,10 +252,12 @@ export const runCli = async (directories: string[], cwd: string, options: CliOpt
logger.trace('options:', options);

if (options.mcp) {
const { runMcpAction } = await import('./actions/mcpAction.js');
return await runMcpAction();
}

if (options.version) {
const { runVersionAction } = await import('./actions/versionAction.js');
await runVersionAction();
return;
}
Expand All @@ -272,17 +269,21 @@ export const runCli = async (directories: string[], cwd: string, options: CliOpt
}

if (options.init) {
const { runInitAction } = await import('./actions/initAction.js');
await runInitAction(cwd, options.global || false);
return;
}

if (options.remote) {
const { runRemoteAction } = await import('./actions/remoteAction.js');
return await runRemoteAction(options.remote, options);
}

// Auto-detect explicit remote URLs (https://, git@, ssh://, git://) in positional arguments
if (directories.length === 1 && isExplicitRemoteUrl(directories[0])) {
// Inline prefix check to avoid loading git-url-parse module for non-remote runs
if (directories.length === 1 && ['https://', 'git@', 'ssh://', 'git://'].some((p) => directories[0].startsWith(p))) {
logger.trace(`Auto-detected remote URL from positional argument: ${directories[0]}`);
const { runRemoteAction } = await import('./actions/remoteAction.js');
return await runRemoteAction(directories[0], options);
}

Expand Down
9 changes: 5 additions & 4 deletions src/cli/cliSpinner.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import logUpdate from 'log-update';
import pc from 'picocolors';
import type { CliOptions } from './types.js';

// Replicate cli-spinners dots animation
const dotsFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
const dotsInterval = 80;

// ANSI escape: erase current line + carriage return (replaces log-update dependency)
const CLEAR_LINE = '\x1B[2K\r';

export class Spinner {
private message: string;
private currentFrame = 0;
Expand All @@ -28,7 +30,7 @@ export class Spinner {
this.interval = setInterval(() => {
this.currentFrame++;
const frame = dotsFrames[this.currentFrame % framesLength];
logUpdate(`${pc.cyan(frame)} ${this.message}`);
process.stderr.write(`${CLEAR_LINE}${pc.cyan(frame)} ${this.message}`);
}, dotsInterval);
}

Expand All @@ -49,8 +51,7 @@ export class Spinner {
clearInterval(this.interval);
this.interval = null;
}
logUpdate(finalMessage);
logUpdate.done();
process.stderr.write(`${CLEAR_LINE}${finalMessage}\n`);
}

succeed(message: string): void {
Expand Down
55 changes: 34 additions & 21 deletions src/core/file/fileTreeGenerate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ export const generateFileTree = (files: string[], emptyDirPaths: string[] = []):
addPathToTree(root, dir, true);
}

// Sort once after tree construction instead of on every treeToString call
sortTreeNodes(root);

return root;
};

Expand Down Expand Up @@ -68,20 +71,43 @@ const sortTreeNodes = (node: TreeNode) => {
}
};

const treeToStringInner = (node: TreeNode, prefix: string, parts: string[]): void => {
for (const child of node.children) {
parts.push(prefix, child.name, child.isDirectory ? '/\n' : '\n');
if (child.isDirectory) {
treeToStringInner(child, `${prefix} `, parts);
}
}
};

export const treeToString = (node: TreeNode, prefix = '', _isRoot = true): string => {
if (_isRoot) {
sortTreeNodes(node);
}
Comment on lines 83 to 86
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The sortTreeNodes(node) call within treeToString is now redundant. The pull request description states that sortTreeNodes was moved to generateFileTree to sort once at build time. Since generateFileTree already sorts the tree, calling sortTreeNodes again here leads to unnecessary double sorting. The _isRoot parameter also becomes unnecessary.

export const treeToString = (node: TreeNode, prefix = ''): string => {
  const parts: string[] = [];
  treeToStringInner(node, prefix, parts);
  return parts.join('');
};

let result = '';
const parts: string[] = [];
treeToStringInner(node, prefix, parts);
return parts.join('');
};

const treeToStringWithLineCountsInner = (
node: TreeNode,
lineCounts: Record<string, number>,
prefix: string,
currentPath: string,
parts: string[],
): void => {
for (const child of node.children) {
result += `${prefix}${child.name}${child.isDirectory ? '/' : ''}\n`;
const childPath = currentPath ? `${currentPath}/${child.name}` : child.name;

if (child.isDirectory) {
result += treeToString(child, `${prefix} `, false);
parts.push(prefix, child.name, '/\n');
treeToStringWithLineCountsInner(child, lineCounts, `${prefix} `, childPath, parts);
} else {
const lineCount = lineCounts[childPath];
const lineCountSuffix = lineCount !== undefined ? ` (${lineCount} lines)` : '';
parts.push(prefix, child.name, lineCountSuffix, '\n');
}
}

return result;
};

/**
Expand All @@ -101,22 +127,9 @@ export const treeToStringWithLineCounts = (
if (_isRoot) {
sortTreeNodes(node);
}
let result = '';

for (const child of node.children) {
const childPath = currentPath ? `${currentPath}/${child.name}` : child.name;

if (child.isDirectory) {
result += `${prefix}${child.name}/\n`;
result += treeToStringWithLineCounts(child, lineCounts, `${prefix} `, childPath, false);
} else {
const lineCount = lineCounts[childPath];
const lineCountSuffix = lineCount !== undefined ? ` (${lineCount} lines)` : '';
result += `${prefix}${child.name}${lineCountSuffix}\n`;
}
}

return result;
const parts: string[] = [];
treeToStringWithLineCountsInner(node, lineCounts, prefix, currentPath, parts);
return parts.join('');
};

export const generateTreeString = (files: string[], emptyDirPaths: string[] = []): string => {
Expand Down
37 changes: 20 additions & 17 deletions src/core/file/truncateBase64.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,18 @@ const TRUNCATION_LENGTH = 32;
const MIN_CHAR_DIVERSITY = 10;
const MIN_CHAR_TYPE_COUNT = 3;

// Pre-compiled regex patterns (hoisted to module scope to avoid recompilation per file)
const dataUriPattern = new RegExp(
`data:([a-zA-Z0-9\\/\\-\\+]+)(;[a-zA-Z0-9\\-=]+)*;base64,([A-Za-z0-9+/=]{${MIN_BASE64_LENGTH_DATA_URI},})`,
'g',
);
const standaloneBase64Pattern = new RegExp(`([A-Za-z0-9+/]{${MIN_BASE64_LENGTH_STANDALONE},}={0,2})`, 'g');
const base64ValidCharsPattern = /^[A-Za-z0-9+/]+=*$/;
const hasNumbersPattern = /[0-9]/;
const hasUpperCasePattern = /[A-Z]/;
const hasLowerCasePattern = /[a-z]/;
const hasSpecialCharsPattern = /[+/]/;

/**
* Truncates base64 encoded data in content to reduce file size
* Detects common base64 patterns like data URIs and standalone base64 strings
Expand All @@ -13,25 +25,16 @@ const MIN_CHAR_TYPE_COUNT = 3;
* @returns Content with base64 data truncated
*/
export const truncateBase64Content = (content: string): string => {
// Pattern to match data URIs (e.g., data:image/png;base64,...)
const dataUriPattern = new RegExp(
`data:([a-zA-Z0-9\\/\\-\\+]+)(;[a-zA-Z0-9\\-=]+)*;base64,([A-Za-z0-9+/=]{${MIN_BASE64_LENGTH_DATA_URI},})`,
'g',
);

// Pattern to match standalone base64 strings
// This matches base64 strings that are likely encoded binary data
const standaloneBase64Pattern = new RegExp(`([A-Za-z0-9+/]{${MIN_BASE64_LENGTH_STANDALONE},}={0,2})`, 'g');

let processedContent = content;

// Replace data URIs
// Reset lastIndex for global regexes before each use
dataUriPattern.lastIndex = 0;
processedContent = processedContent.replace(dataUriPattern, (_match, mimeType, params, base64Data) => {
const preview = base64Data.substring(0, TRUNCATION_LENGTH);
return `data:${mimeType}${params || ''};base64,${preview}...`;
});

// Replace standalone base64 strings
standaloneBase64Pattern.lastIndex = 0;
processedContent = processedContent.replace(standaloneBase64Pattern, (match, base64String) => {
// Check if this looks like actual base64 (not just a long string)
if (isLikelyBase64(base64String)) {
Expand All @@ -52,7 +55,7 @@ export const truncateBase64Content = (content: string): string => {
*/
function isLikelyBase64(str: string): boolean {
// Check for valid base64 characters only
if (!/^[A-Za-z0-9+/]+=*$/.test(str)) {
if (!base64ValidCharsPattern.test(str)) {
return false;
}

Expand All @@ -64,10 +67,10 @@ function isLikelyBase64(str: string): boolean {

// Additional check: base64 encoded binary data typically has good character distribution
// Must have at least MIN_CHAR_TYPE_COUNT of the 4 character types (numbers, uppercase, lowercase, special)
const hasNumbers = /[0-9]/.test(str);
const hasUpperCase = /[A-Z]/.test(str);
const hasLowerCase = /[a-z]/.test(str);
const hasSpecialChars = /[+/]/.test(str);
const hasNumbers = hasNumbersPattern.test(str);
const hasUpperCase = hasUpperCasePattern.test(str);
const hasLowerCase = hasLowerCasePattern.test(str);
const hasSpecialChars = hasSpecialCharsPattern.test(str);

const charTypeCount = [hasNumbers, hasUpperCase, hasLowerCase, hasSpecialChars].filter(Boolean).length;

Expand Down
28 changes: 23 additions & 5 deletions src/core/git/gitRepositoryHandle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,36 @@ export const getFileChangeCount = async (
}
};

// Promise-based cache to deduplicate concurrent isGitRepository calls
// (e.g., when getGitDiffs and getGitLogs run in parallel via Promise.all)
const isGitRepoCache = new Map<string, Promise<boolean>>();

export const isGitRepository = async (
directory: string,
deps = {
execGitRevParse,
},
): Promise<boolean> => {
try {
await deps.execGitRevParse(directory);
return true;
} catch {
return false;
// Only use cache with default deps (skip for test mocks)
const useCache = deps.execGitRevParse === execGitRevParse;

if (useCache) {
const cached = isGitRepoCache.get(directory);
if (cached) {
return cached;
}
}

const promise = deps.execGitRevParse(directory).then(
() => true,
() => false,
);

if (useCache) {
isGitRepoCache.set(directory, promise);
}

return promise;
};

export const isGitInstalled = async (
Expand Down
15 changes: 11 additions & 4 deletions src/core/output/outputGenerate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,24 @@ const calculateMarkdownDelimiter = (files: ReadonlyArray<ProcessedFile>): string
return '`'.repeat(Math.max(3, maxBackticks + 1));
};

const countNewlines = (str: string): number => {
let count = 0;
let pos = str.indexOf('\n');
while (pos !== -1) {
count++;
pos = str.indexOf('\n', pos + 1);
}
return count;
};

const calculateFileLineCounts = (processedFiles: ProcessedFile[]): Record<string, number> => {
const lineCounts: Record<string, number> = {};
for (const file of processedFiles) {
// Count lines: empty files have 0 lines, otherwise count newlines + 1
// (unless the content ends with a newline, in which case the last "line" is empty)
const content = file.content;
if (content.length === 0) {
lineCounts[file.path] = 0;
} else {
// Count actual lines (text editor style: number of \n + 1, but trailing \n doesn't add extra line)
const newlineCount = (content.match(/\n/g) || []).length;
const newlineCount = countNewlines(content);
lineCounts[file.path] = content.endsWith('\n') ? newlineCount : newlineCount + 1;
}
}
Expand Down
13 changes: 6 additions & 7 deletions src/core/packager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,13 +105,12 @@ export const pack = async (
const rawFiles = collectResults.flatMap((curr) => curr.rawFiles);
const allSkippedFiles = collectResults.flatMap((curr) => curr.skippedFiles);

// Get git diffs if enabled - run this before security check
progressCallback('Getting git diffs...');
const gitDiffResult = await deps.getGitDiffs(rootDirs, config);

// Get git logs if enabled - run this before security check
progressCallback('Getting git logs...');
const gitLogResult = await deps.getGitLogs(rootDirs, config);
// Get git diffs and logs in parallel - both are independent subprocess calls
progressCallback('Getting git information...');
const [gitDiffResult, gitLogResult] = await Promise.all([
deps.getGitDiffs(rootDirs, config),
deps.getGitLogs(rootDirs, config),
]);

// Run security check and get filtered safe files
const { safeFilePaths, safeRawFiles, suspiciousFilesResults, suspiciousGitDiffResults, suspiciousGitLogResults } =
Expand Down
6 changes: 4 additions & 2 deletions src/core/security/filterOutUntrustedFiles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,7 @@ import type { SuspiciousFileResult } from './securityCheck.js';
export const filterOutUntrustedFiles = (
rawFiles: RawFile[],
suspiciousFilesResults: SuspiciousFileResult[],
): RawFile[] =>
rawFiles.filter((rawFile) => !suspiciousFilesResults.some((result) => result.filePath === rawFile.path));
): RawFile[] => {
const suspiciousPaths = new Set(suspiciousFilesResults.map((result) => result.filePath));
return rawFiles.filter((rawFile) => !suspiciousPaths.has(rawFile.path));
};
Loading
Loading