diff --git a/packages/core/src/utils/filesearch/crawler.test.ts b/packages/core/src/utils/filesearch/crawler.test.ts index d4063df7ca8..f63e2f163b9 100644 --- a/packages/core/src/utils/filesearch/crawler.test.ts +++ b/packages/core/src/utils/filesearch/crawler.test.ts @@ -7,8 +7,13 @@ import { describe, it, expect, afterEach, vi, beforeEach } from 'vitest'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import * as childProcess from 'node:child_process'; import * as cache from './crawlCache.js'; -import { crawl } from './crawler.js'; +import { + crawl, + __setCommandRunnerForTests, + __resetCrawlerStateForTests, +} from './crawler.js'; import { createTmpDir, cleanupTmpDir, @@ -16,12 +21,54 @@ import { import type { Ignore } from './ignore.js'; import { loadIgnoreRules } from './ignore.js'; +async function runExecFile( + command: string, + args: string[], + cwd: string, +): Promise { + await new Promise((resolve, reject) => { + childProcess.execFile( + command, + args, + { cwd, windowsHide: true }, + (error) => { + if (error) { + reject(error); + return; + } + resolve(); + }, + ); + }); +} + +async function initGitRepo(dir: string): Promise { + await runExecFile('git', ['init'], dir); + await runExecFile('git', ['add', '.'], dir); + await runExecFile( + 'git', + [ + '-c', + 'user.name=Qwen Test', + '-c', + 'user.email=qwen-test@example.com', + 'commit', + '--no-gpg-sign', + '-m', + 'init', + ], + dir, + ); +} + describe('crawler', () => { let tmpDir: string; afterEach(async () => { if (tmpDir) { await cleanupTmpDir(tmpDir); } + __setCommandRunnerForTests(); + __resetCrawlerStateForTests(); vi.restoreAllMocks(); }); @@ -575,6 +622,37 @@ describe('crawler', () => { ]), ); }); + + it('should treat maxDepth as relative to the crawl directory', async () => { + await initGitRepo(tmpDir); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: path.join(tmpDir, 'level1'), + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + maxDepth: 0, + }); + + expect(results).toEqual( + expect.arrayContaining([ + '.', + 'level1/', + 'level1/file-level1.txt', + 'level1/level2/', + ]), + ); + expect(results).not.toContain('level1/level2/file-level2.txt'); + expect(results).not.toContain('level1/level2/level3/'); + }); }); describe('with maxFiles', () => { @@ -678,4 +756,618 @@ describe('crawler', () => { expect(results).toEqual(expect.arrayContaining(['.', 'a.txt', 'b.txt'])); }); }); + + describe('two-tier strategy: git ls-files + ripgrep fallback', () => { + it('should use git ls-files in a git repo', async () => { + tmpDir = await createTmpDir({ + 'file1.js': '', + src: ['file2.js'], + }); + await initGitRepo(tmpDir); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }); + + expect(results).toEqual( + expect.arrayContaining(['.', 'src/', 'file1.js', 'src/file2.js']), + ); + }); + + it('should not include tracked files deleted from the working tree', async () => { + tmpDir = await createTmpDir({ + 'alive.txt': '', + 'deleted.txt': '', + }); + await initGitRepo(tmpDir); + await fs.unlink(path.join(tmpDir, 'deleted.txt')); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }); + + expect(results).toContain('alive.txt'); + expect(results).not.toContain('deleted.txt'); + }); + + it('should include tracked dangling symlinks', async () => { + if (process.platform === 'win32') { + return; + } + + tmpDir = await createTmpDir({ + 'alive.txt': '', + }); + await fs.symlink( + 'missing-target.txt', + path.join(tmpDir, 'broken-link.txt'), + ); + await initGitRepo(tmpDir); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }); + + expect(results).toContain('alive.txt'); + expect(results).toContain('broken-link.txt'); + }); + + it('should fall back to fdir when not in a git repo and ripgrep unavailable', async () => { + __setCommandRunnerForTests(async (command) => { + if (command === 'git' || command === 'rg') { + return { success: false, lines: [] }; + } + return { success: false, lines: [] }; + }); + + tmpDir = await createTmpDir({ + 'index.js': '', + lib: ['util.js'], + }); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }); + + expect(results).toEqual( + expect.arrayContaining(['.', 'lib/', 'index.js', 'lib/util.js']), + ); + }); + + it('should keep ignore-file handling enabled in ripgrep fallback', async () => { + const rgArgsSeen: string[][] = []; + + __setCommandRunnerForTests(async (command, args) => { + if (command === 'git') { + return { success: false, lines: [] }; + } + + if (command === 'rg') { + rgArgsSeen.push(args); + return { success: true, lines: ['index.js'] }; + } + + return { success: false, lines: [] }; + }); + + tmpDir = await createTmpDir({ + 'index.js': '', + }); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }); + + expect(results).toEqual(expect.arrayContaining(['.', 'index.js'])); + expect(rgArgsSeen).toHaveLength(1); + expect(rgArgsSeen[0]).toEqual([ + '--files', + '--no-require-git', + '--hidden', + ]); + expect(rgArgsSeen[0]).not.toContain('--no-ignore'); + }); + + it('should respect maxDepth on git ls-files path', async () => { + tmpDir = await createTmpDir({ + root: ['top.js'], + nested: { + deep: ['file.js'], + }, + }); + await initGitRepo(tmpDir); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const results = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + maxDepth: 0, + }); + + expect(results).toEqual( + expect.arrayContaining(['.', 'root/', 'nested/']), + ); + expect(results).not.toContain('root/top.js'); + expect(results).not.toContain('nested/deep/'); + expect(results).not.toContain('nested/deep/file.js'); + }); + + it('should avoid enumerating gitignored untracked files on git path', async () => { + tmpDir = await createTmpDir({ + '.gitignore': '*.log', + 'keep.log': '', + 'keep.txt': '', + }); + await initGitRepo(tmpDir); + + const withoutGitignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const withoutGitignoreResults = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore: withoutGitignore, + cache: false, + cacheTtl: 0, + }); + expect(withoutGitignoreResults).not.toContain('keep.log'); + expect(withoutGitignoreResults).toContain('keep.txt'); + + const withGitignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: true, + useQwenignore: false, + ignoreDirs: [], + }); + const withGitignoreResults = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore: withGitignore, + cache: false, + cacheTtl: 0, + }); + expect(withGitignoreResults).not.toContain('keep.log'); + expect(withGitignoreResults).toContain('keep.txt'); + }); + }); + + describe('throttling', () => { + beforeEach(() => { + cache.clear(); + }); + + it('should not re-crawl within throttle window', async () => { + tmpDir = await createTmpDir({ 'file1.js': '' }); + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const results1 = await crawl(options); + expect(results1).toContain('file1.js'); + + const results2 = await crawl(options); + expect(results2).toContain('file1.js'); + }); + + it('should skip untracked refresh while reusing throttled git results', async () => { + tmpDir = await createTmpDir({ + 'tracked.js': '', + }); + + let listOthersCalls = 0; + __setCommandRunnerForTests(async (command, args) => { + if (command !== 'git') { + return { success: false, lines: [] }; + } + + if (args[0] === 'rev-parse' && args.includes('--show-toplevel')) { + return { success: true, lines: [tmpDir] }; + } + + if (args[0] === 'ls-files' && args.includes('--cached')) { + return { success: true, lines: ['tracked.js'] }; + } + + if (args[0] === 'ls-files' && args.includes('--others')) { + listOthersCalls += 1; + return { success: true, lines: [] }; + } + + return { success: false, lines: [] }; + }); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const first = await crawl(options); + expect(first).toContain('tracked.js'); + expect(first).not.toContain('new-untracked.js'); + + await fs.writeFile(path.join(tmpDir, 'new-untracked.js'), ''); + + const second = await crawl(options); + expect(second).toContain('tracked.js'); + expect(second).not.toContain('new-untracked.js'); + expect(listOthersCalls).toBe(1); + }); + + it('should not reuse throttled snapshot when untracked listing fails transiently', async () => { + tmpDir = await createTmpDir({ + 'tracked.js': '', + }); + + let listOthersAttempt = 0; + let listCachedCalls = 0; + + __setCommandRunnerForTests(async (command, args) => { + if (command !== 'git') { + return { success: false, lines: [] }; + } + + if (args[0] === 'rev-parse' && args.includes('--show-toplevel')) { + return { success: true, lines: [tmpDir] }; + } + + if (args[0] === 'ls-files' && args.includes('--cached')) { + listCachedCalls += 1; + return { success: true, lines: ['tracked.js'] }; + } + + if (args[0] === 'ls-files' && args.includes('--others')) { + listOthersAttempt += 1; + if (listOthersAttempt === 1) { + return { success: false, lines: [] }; + } + return { success: true, lines: ['new-untracked.js'] }; + } + + return { success: false, lines: [] }; + }); + + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const first = await crawl(options); + expect(first).toContain('tracked.js'); + expect(first).not.toContain('new-untracked.js'); + + await fs.writeFile(path.join(tmpDir, 'new-untracked.js'), ''); + + const second = await crawl(options); + expect(second).toContain('tracked.js'); + expect(second).toContain('new-untracked.js'); + + // The first crawl should not cache a throttled snapshot when --others failed, + // so the second crawl must run git ls-files --cached again. + expect(listCachedCalls).toBe(2); + }); + + it('should throttle re-crawl on non-git fallback paths until the window expires', async () => { + __setCommandRunnerForTests(async (command) => { + if (command === 'git' || command === 'rg') { + return { success: false, lines: [] }; + } + return { success: false, lines: [] }; + }); + + tmpDir = await createTmpDir({ 'file1.js': '' }); + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + vi.useFakeTimers(); + try { + const first = await crawl(options); + expect(first).toContain('file1.js'); + + await fs.writeFile(path.join(tmpDir, 'file2.js'), ''); + + const second = await crawl(options); + expect(second).toContain('file1.js'); + expect(second).not.toContain('file2.js'); + + await vi.advanceTimersByTimeAsync(6000); + + const third = await crawl(options); + expect(third).toContain('file1.js'); + expect(third).toContain('file2.js'); + } finally { + vi.useRealTimers(); + } + }); + + it('should not reuse throttled snapshot across different maxFiles values', async () => { + __setCommandRunnerForTests(async (command) => { + if (command === 'git' || command === 'rg') { + return { success: false, lines: [] }; + } + return { success: false, lines: [] }; + }); + + tmpDir = await createTmpDir({ + 'a.js': '', + 'b.js': '', + 'c.js': '', + }); + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + + const first = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + maxFiles: 2, + }); + + const second = await crawl({ + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + maxFiles: 100, + }); + + expect(second.length).toBeGreaterThan(first.length); + expect(second).toEqual( + expect.arrayContaining(['.', 'a.js', 'b.js', 'c.js']), + ); + }); + }); + + describe('mtime-based change detection', () => { + beforeEach(() => { + cache.clear(); + }); + + it('should re-crawl when git index mtime changes', async () => { + tmpDir = await createTmpDir({ 'file1.js': '' }); + const ignore = loadIgnoreRules({ + projectRoot: tmpDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: tmpDir, + cwd: tmpDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const results1 = await crawl(options); + expect(results1.length).toBeGreaterThan(0); + + await fs.writeFile(path.join(tmpDir, 'file2.js'), ''); + + const results2 = await crawl(options); + expect(results2.length).toBeGreaterThanOrEqual(results1.length); + }); + + it('should detect git index changes when crawling a subdirectory', async () => { + tmpDir = await createTmpDir({ + nested: { + 'tracked.txt': '', + }, + }); + await initGitRepo(tmpDir); + + const nestedDir = path.join(tmpDir, 'nested'); + const ignore = loadIgnoreRules({ + projectRoot: nestedDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: nestedDir, + cwd: nestedDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const first = await crawl(options); + expect(first).toContain('tracked.txt'); + expect(first).not.toContain('new-tracked.txt'); + + await fs.writeFile(path.join(nestedDir, 'new-tracked.txt'), ''); + await runExecFile('git', ['add', 'nested/new-tracked.txt'], tmpDir); + const indexPath = path.join(tmpDir, '.git', 'index'); + const futureTime = new Date(Date.now() + 60_000); + await fs.utimes(indexPath, futureTime, futureTime); + + const second = await crawl(options); + expect(second).toContain('tracked.txt'); + expect(second).toContain('new-tracked.txt'); + }); + + it('should re-crawl git worktrees when the gitdir index changes', async () => { + const worktreeDir = path.join(tmpDir, 'worktree'); + const gitDir = path.join(tmpDir, 'gitdir'); + + await fs.mkdir(worktreeDir, { recursive: true }); + await fs.mkdir(gitDir, { recursive: true }); + await fs.writeFile(path.join(gitDir, 'index'), 'initial'); + await fs.writeFile(path.join(worktreeDir, '.git'), 'gitdir: ../gitdir\n'); + await fs.writeFile(path.join(worktreeDir, 'tracked.txt'), ''); + + let includeExtraFile = false; + __setCommandRunnerForTests( + async ( + command: string, + args: string[], + cwd: string, + ): Promise<{ success: boolean; lines: string[] }> => { + if (command !== 'git') { + return { success: false, lines: [] }; + } + + if (args[0] === 'rev-parse' && args.includes('--show-toplevel')) { + expect(cwd).toBe(worktreeDir); + return { success: true, lines: [worktreeDir] }; + } + + if (args[0] === 'ls-files' && args.includes('--cached')) { + return { + success: true, + lines: includeExtraFile + ? ['tracked.txt', 'new-file.txt'] + : ['tracked.txt'], + }; + } + + if (args[0] === 'ls-files' && args.includes('--others')) { + return { success: true, lines: [] }; + } + + return { success: false, lines: [] }; + }, + ); + + const ignore = loadIgnoreRules({ + projectRoot: worktreeDir, + useGitignore: false, + useQwenignore: false, + ignoreDirs: [], + }); + const options = { + crawlDirectory: worktreeDir, + cwd: worktreeDir, + ignore, + cache: false, + cacheTtl: 0, + }; + + const first = await crawl(options); + expect(first).toEqual(expect.arrayContaining(['.', 'tracked.txt'])); + expect(first).not.toContain('new-file.txt'); + + includeExtraFile = true; + await fs.writeFile(path.join(worktreeDir, 'new-file.txt'), ''); + const futureTime = new Date(Date.now() + 60_000); + await fs.utimes(path.join(gitDir, 'index'), futureTime, futureTime); + + const second = await crawl(options); + expect(second).toEqual( + expect.arrayContaining(['.', 'tracked.txt', 'new-file.txt']), + ); + }); + }); }); diff --git a/packages/core/src/utils/filesearch/crawler.ts b/packages/core/src/utils/filesearch/crawler.ts index 0fdf282b335..e6ccbed71b6 100644 --- a/packages/core/src/utils/filesearch/crawler.ts +++ b/packages/core/src/utils/filesearch/crawler.ts @@ -5,6 +5,8 @@ */ import path from 'node:path'; +import fs from 'node:fs'; +import { execFile } from 'node:child_process'; import { fdir } from 'fdir'; import type { Ignore } from './ignore.js'; import * as cache from './crawlCache.js'; @@ -25,28 +27,623 @@ export interface CrawlOptions { cacheTtl: number; } -function toPosixPath(p: string) { +function toPosixPath(p: string): string { return p.split(path.sep).join(path.posix.sep); } -export async function crawl(options: CrawlOptions): Promise { - if (options.cache) { - const cacheKey = cache.getCacheKey( - options.crawlDirectory, - options.ignore.getFingerprint(), - options.maxDepth, - options.maxFiles, +const THROTTLE_MS = 5_000; +const lastRebuildTime = new Map(); + +function getStateKey(options: CrawlOptions): string { + return [ + normalizePath(options.crawlDirectory), + normalizePath(options.cwd), + options.ignore.getFingerprint(), + options.maxDepth === undefined ? 'undefined' : String(options.maxDepth), + options.maxFiles === undefined ? 'undefined' : String(options.maxFiles), + ].join('|'); +} + +function isThrottled(stateKey: string): boolean { + const last = lastRebuildTime.get(stateKey); + if (last === undefined) return false; + return Date.now() - last < THROTTLE_MS; +} + +function recordRebuild(stateKey: string): void { + lastRebuildTime.set(stateKey, Date.now()); +} + +interface ChangeState { + gitRootMtimeMs: number | null; + untrackedFingerprint: string | null; + fileList: string[]; +} + +const changeStateMap = new Map(); + +function resolveGitDir(crawlDirectory: string): string | null { + let current = crawlDirectory; + while (current) { + const gitPath = path.join(current, '.git'); + + try { + const stat = fs.statSync(gitPath); + + if (stat.isDirectory()) { + return gitPath; + } + + if (stat.isFile()) { + const contents = fs.readFileSync(gitPath, 'utf8').trim(); + const match = contents.match(/^gitdir:\s*(.+)$/i); + if (!match) { + return null; + } + + const resolvedGitDir = match[1].trim(); + return path.isAbsolute(resolvedGitDir) + ? resolvedGitDir + : path.resolve(current, resolvedGitDir); + } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ENOTDIR') { + return null; + } + // Keep walking ancestors when .git is simply missing here. + } + + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + + return null; +} + +function getGitRootMtime(crawlDirectory: string): number | null { + try { + const gitDir = resolveGitDir(crawlDirectory); + if (!gitDir) { + return null; + } + + const indexPath = path.join(gitDir, 'index'); + const indexStat = fs.statSync(indexPath); + return indexStat.mtimeMs; + } catch { + // Ignore errors when .git metadata or index file doesn't exist. + } + return null; +} + +function hasFileListChanged(stateKey: string, crawlDirectory: string): boolean { + const currentMtime = getGitRootMtime(crawlDirectory); + const state = changeStateMap.get(stateKey); + + if (!state) return true; + + if (currentMtime !== null && state.gitRootMtimeMs !== null) { + return currentMtime > state.gitRootMtimeMs || !isThrottled(stateKey); + } + + // For non-git paths, we can only rely on time-based throttling. + if (currentMtime === null && state.gitRootMtimeMs === null) { + return !isThrottled(stateKey); + } + + return true; +} + +function updateChangeState( + stateKey: string, + crawlDirectory: string, + fileList: string[], + untrackedFiles?: string[], +): void { + const mtime = getGitRootMtime(crawlDirectory); + changeStateMap.set(stateKey, { + gitRootMtimeMs: mtime, + untrackedFingerprint: + untrackedFiles === undefined + ? null + : computeLinesFingerprint(untrackedFiles), + fileList, + }); +} + +function computeLinesFingerprint(lines: string[]): string { + let hash = 5381; + for (const line of lines) { + for (let i = 0; i < line.length; i++) { + hash = ((hash << 5) + hash + line.charCodeAt(i)) >>> 0; + } + hash = ((hash << 5) + hash + 10) >>> 0; + } + return `${lines.length}:${hash}`; +} + +interface CommandResult { + success: boolean; + lines: string[]; +} + +function runCommand( + command: string, + args: string[], + cwd: string, + timeoutMs: number = 20_000, +): Promise { + return new Promise((resolve) => { + const child = execFile( + command, + args, + { cwd, timeout: timeoutMs, maxBuffer: 20_000_000, windowsHide: true }, + (error, stdout = '') => { + if (error) { + resolve({ success: false, lines: [] }); + return; + } + const lines = stdout + .split('\n') + .map((l) => l) + .filter((l) => l.length > 0); + resolve({ success: true, lines }); + }, ); - const cachedResults = cache.read(cacheKey); + child.on('error', () => resolve({ success: false, lines: [] })); + }); +} - if (cachedResults) { - return cachedResults; +type CommandRunner = typeof runCommand; +let commandRunner: CommandRunner = runCommand; + +export function __setCommandRunnerForTests(runner?: CommandRunner): void { + commandRunner = runner ?? runCommand; +} + +export function __resetCrawlerStateForTests(): void { + lastRebuildTime.clear(); + changeStateMap.clear(); +} + +function normalizePath(p: string): string { + return toPosixPath(p); +} + +function normalizeForComparison(p: string): string { + const normalized = normalizePath(p); + if (/^[A-Z]:/.test(normalized)) { + return `${normalized[0].toLowerCase()}${normalized.slice(1)}`; + } + return normalized; +} + +function canonicalizePath(p: string): string { + try { + return fs.realpathSync.native(p); + } catch { + return path.resolve(p); + } +} + +function getPosixRelative(from: string, to: string): string { + const canonicalFrom = normalizeForComparison(canonicalizePath(from)); + const canonicalTo = normalizeForComparison(canonicalizePath(to)); + const relative = path.posix.relative(canonicalFrom, canonicalTo); + return relative === '' ? '.' : relative; +} + +function isValidIgnorePath(relativePath: string): boolean { + if (!relativePath || relativePath === '.') { + return false; + } + + if (path.posix.isAbsolute(relativePath)) { + return false; + } + + return ( + relativePath !== '..' && + !relativePath.startsWith('../') && + !relativePath.includes('/../') + ); +} + +function toIgnoreRelativePath( + baseDir: string, + candidatePath: string, +): string | null { + const absoluteCandidate = path.isAbsolute(candidatePath) + ? candidatePath + : path.join(baseDir, candidatePath); + const relativePath = getPosixRelative(baseDir, absoluteCandidate); + return isValidIgnorePath(relativePath) ? relativePath : null; +} + +function getEntryDepth(entry: string): number { + if (entry === '.') { + return -1; + } + + const withoutTrailingSlash = entry.endsWith('/') ? entry.slice(0, -1) : entry; + if (withoutTrailingSlash.length === 0) { + return -1; + } + + return withoutTrailingSlash.split('/').length - 1; +} + +function stripCrawlDirectoryPrefix( + entry: string, + relativeToCrawlDir: string, +): string { + if ( + entry === '.' || + relativeToCrawlDir === '' || + relativeToCrawlDir === '.' + ) { + return entry; + } + + const prefix = relativeToCrawlDir.endsWith('/') + ? relativeToCrawlDir + : `${relativeToCrawlDir}/`; + + if (entry === relativeToCrawlDir) { + return '.'; + } + + if (entry.startsWith(prefix)) { + return entry.slice(prefix.length) || '.'; + } + + return entry; +} + +function applyMaxDepthLimit( + results: string[], + maxDepth?: number, + relativeToCrawlDir?: string, +): string[] { + if (maxDepth === undefined) { + return results; + } + + return results.filter((entry) => { + if (entry === '.') { + return true; + } + + const crawlRootRelativeEntry = relativeToCrawlDir + ? stripCrawlDirectoryPrefix(entry, relativeToCrawlDir) + : entry; + + return getEntryDepth(crawlRootRelativeEntry) <= maxDepth; + }); +} + +function isUnderIgnoredDirectory( + filePath: string, + dirFilter: (dirPath: string) => boolean, +): boolean { + const parts = filePath.split('/'); + let current = ''; + + for (let i = 0; i < parts.length - 1; i++) { + current = current ? `${current}/${parts[i]}` : parts[i]; + if (dirFilter(`${current}/`)) { + return true; } } - const posixCwd = toPosixPath(options.cwd); - const posixCrawlDirectory = toPosixPath(options.crawlDirectory); - const relativeToCrawlDir = path.posix.relative(posixCwd, posixCrawlDirectory); + return false; +} + +function applyFilters( + results: string[], + options: CrawlOptions, + relativeToCrawlDir?: string, +): string[] { + const depthFiltered = applyMaxDepthLimit( + results, + options.maxDepth, + relativeToCrawlDir, + ); + const dirFilter = options.ignore.getDirectoryFilter(); + const fileFilter = options.ignore.getFileFilter(); + + return depthFiltered.filter((p) => { + if (p === '.') return true; + + if (p.endsWith('/')) { + if (!isValidIgnorePath(p.slice(0, -1))) { + return false; + } + return !dirFilter(p); + } + + if (!isValidIgnorePath(p)) { + return false; + } + + if (isUnderIgnoredDirectory(p, dirFilter)) { + return false; + } + + return !fileFilter(p); + }); +} + +const YIELD_INTERVAL = 1000; + +async function maybeYield(index: number): Promise { + if (index % YIELD_INTERVAL === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } +} + +async function findGitRoot(dir: string): Promise { + const result = await commandRunner( + 'git', + ['rev-parse', '--show-toplevel'], + dir, + 5_000, + ); + if (!result.success || result.lines.length === 0) return null; + return normalizePath(result.lines[0]); +} + +function shouldIncludeFile( + filePath: string, + dirFilter: (dirPath: string) => boolean, + fileFilter: (filePath: string) => boolean, +): boolean { + if (!isValidIgnorePath(filePath)) { + return false; + } + + if (isUnderIgnoredDirectory(filePath, dirFilter)) { + return false; + } + + if (fileFilter(filePath)) { + return false; + } + + return true; +} + +function hasReachedFileBudget( + fileSet: Set, + maxFiles?: number, +): boolean { + return maxFiles !== undefined && fileSet.size >= maxFiles; +} + +function existingTrackedPath(gitRoot: string, normalizedFile: string): boolean { + try { + fs.lstatSync(path.join(gitRoot, normalizedFile)); + return true; + } catch { + return false; + } +} + +async function listUntrackedFiles( + gitRoot: string, + relativeToGitRoot: string, +): Promise { + const untrackedArgs = ['ls-files', '--others', '--exclude-standard']; + if (relativeToGitRoot && relativeToGitRoot !== '.') { + untrackedArgs.push(relativeToGitRoot); + } + + const untrackedResult = await commandRunner( + 'git', + untrackedArgs, + gitRoot, + 10_000, + ); + if (!untrackedResult.success) { + return null; + } + + return untrackedResult.lines.map((file) => normalizePath(file)); +} + +async function hasUntrackedFilesChanged( + state: ChangeState, + crawlDirectory: string, +): Promise { + if (state.untrackedFingerprint === null) { + return false; + } + + const gitRoot = await findGitRoot(crawlDirectory); + if (!gitRoot) { + return true; + } + + const relativeToGitRoot = getPosixRelative(gitRoot, crawlDirectory); + const untrackedFiles = await listUntrackedFiles(gitRoot, relativeToGitRoot); + if (untrackedFiles === null) { + return true; + } + + return computeLinesFingerprint(untrackedFiles) !== state.untrackedFingerprint; +} + +async function crawlWithGitLsFiles( + stateKey: string, + crawlDirectory: string, + cwd: string, + options: CrawlOptions, +): Promise<{ success: boolean; files: string[]; isGitRepo: boolean }> { + const gitRoot = await findGitRoot(crawlDirectory); + if (!gitRoot) { + return { success: false, files: [], isGitRepo: false }; + } + + const relativeToCrawlDir = getPosixRelative(cwd, crawlDirectory); + const relativeToGitRoot = getPosixRelative(gitRoot, crawlDirectory); + const dirFilter = options.ignore.getDirectoryFilter(); + const fileFilter = options.ignore.getFileFilter(); + + const trackedArgs = ['ls-files', '--cached']; + if (relativeToGitRoot && relativeToGitRoot !== '.') { + trackedArgs.push(relativeToGitRoot); + } + const trackedResult = await commandRunner( + 'git', + trackedArgs, + gitRoot, + 20_000, + ); + if (!trackedResult.success) { + return { success: false, files: [], isGitRepo: true }; + } + + const untrackedFiles = await listUntrackedFiles(gitRoot, relativeToGitRoot); + + const fileSet = new Set(); + let count = 0; + + for (const file of trackedResult.lines) { + if (hasReachedFileBudget(fileSet, options.maxFiles)) { + break; + } + + await maybeYield(count++); + const normalizedFile = normalizePath(file); + if (!existingTrackedPath(gitRoot, normalizedFile)) { + continue; + } + + const fullPath = + relativeToGitRoot && relativeToGitRoot !== '.' + ? path.posix.join( + relativeToCrawlDir, + normalizedFile.slice(relativeToGitRoot.length + 1), + ) + : path.posix.join(relativeToCrawlDir, normalizedFile); + + if (!shouldIncludeFile(fullPath, dirFilter, fileFilter)) { + continue; + } + + fileSet.add(fullPath); + } + + if (untrackedFiles !== null) { + for (const normalizedFile of untrackedFiles) { + if (hasReachedFileBudget(fileSet, options.maxFiles)) { + break; + } + + await maybeYield(count++); + const fullPath = + relativeToGitRoot && relativeToGitRoot !== '.' + ? path.posix.join( + relativeToCrawlDir, + normalizedFile.slice(relativeToGitRoot.length + 1), + ) + : path.posix.join(relativeToCrawlDir, normalizedFile); + + if (!shouldIncludeFile(fullPath, dirFilter, fileFilter)) { + continue; + } + + if (!fileSet.has(fullPath)) { + fileSet.add(fullPath); + } + } + } + + const results = buildResultsFromFileSet(fileSet); + const filteredResults = applyFilters(results, options, relativeToCrawlDir); + + // Avoid caching a partial git snapshot when untracked listing fails. + // A missing untracked snapshot can hide files during the throttle window. + if (untrackedFiles !== null) { + updateChangeState( + stateKey, + crawlDirectory, + filteredResults, + untrackedFiles, + ); + recordRebuild(stateKey); + } + + return { success: true, files: filteredResults, isGitRepo: true }; +} + +function buildResultsFromFileSet(files: Set): string[] { + const dirSet = new Set(); + for (const file of files) { + const parts = file.split('/'); + let current = ''; + for (let i = 0; i < parts.length - 1; i++) { + current = current ? current + '/' + parts[i] : parts[i]; + dirSet.add(current + '/'); + } + } + return ['.', ...Array.from(dirSet), ...Array.from(files)]; +} + +async function crawlWithRipgrep( + stateKey: string, + crawlDirectory: string, + cwd: string, + options: CrawlOptions, +): Promise<{ success: boolean; files: string[] }> { + const rgResult = await commandRunner( + 'rg', + ['--files', '--no-require-git', '--hidden'], + crawlDirectory, + 20_000, + ); + + if (!rgResult.success) { + return { success: false, files: [] }; + } + + const relativeToCrawlDir = getPosixRelative(cwd, crawlDirectory); + const dirFilter = options.ignore.getDirectoryFilter(); + const fileFilter = options.ignore.getFileFilter(); + + const fileSet = new Set(); + let count = 0; + for (const file of rgResult.lines) { + if (hasReachedFileBudget(fileSet, options.maxFiles)) { + break; + } + + await maybeYield(count++); + const normalizedFile = normalizePath(file); + + const fullPath = path.posix.join(relativeToCrawlDir, normalizedFile); + if (!shouldIncludeFile(fullPath, dirFilter, fileFilter)) { + continue; + } + + fileSet.add(fullPath); + } + + const results = buildResultsFromFileSet(fileSet); + const filteredResults = applyFilters(results, options, relativeToCrawlDir); + + updateChangeState(stateKey, crawlDirectory, filteredResults); + recordRebuild(stateKey); + return { success: true, files: filteredResults }; +} + +async function crawlWithFdir(options: CrawlOptions): Promise { + const relativeToCrawlDir = getPosixRelative( + options.cwd, + options.crawlDirectory, + ); let results: string[]; try { @@ -55,17 +652,23 @@ export async function crawl(options: CrawlOptions): Promise { const api = new fdir() .withRelativePaths() .withDirs() - .withPathSeparator('/') // Always use unix style paths + .withPathSeparator('/') .exclude((_, dirPath) => { - const relativePath = path.posix.relative(posixCrawlDirectory, dirPath); + const relativePath = toIgnoreRelativePath( + options.crawlDirectory, + dirPath, + ); + if (!relativePath) { + return false; + } return dirFilter(`${relativePath}/`); }) .filter((filePath, isDirectory) => { - // Directories are already handled by the exclude() callback above. if (isDirectory) return true; - // Apply file-level ignore patterns (e.g. *.log, *.map) during the - // crawl so they don't consume the maxFiles budget. const cwdRelative = path.posix.join(relativeToCrawlDir, filePath); + if (!isValidIgnorePath(cwdRelative)) { + return false; + } return !fileFilter(cwdRelative); }); @@ -78,14 +681,100 @@ export async function crawl(options: CrawlOptions): Promise { } results = await api.crawl(options.crawlDirectory).withPromise(); - } catch (_e) { - // The directory probably doesn't exist. + } catch { return []; } - const relativeToCwdResults = results.map((p) => - path.posix.join(relativeToCrawlDir, p), + return results.map((p) => path.posix.join(relativeToCrawlDir, p)); +} + +export async function crawl(options: CrawlOptions): Promise { + const stateKey = getStateKey(options); + + if (options.cache) { + const cacheKey = cache.getCacheKey( + options.crawlDirectory, + options.ignore.getFingerprint(), + options.maxDepth, + options.maxFiles, + ); + const cachedResults = cache.read(cacheKey); + if (cachedResults) { + return cachedResults; + } + } + + if (!options.cache) { + const needReCrawl = hasFileListChanged(stateKey, options.crawlDirectory); + + if (!needReCrawl) { + const state = changeStateMap.get(stateKey); + if (state) { + if (isThrottled(stateKey)) { + return applyMaxFilesLimit(state.fileList, options.maxFiles); + } + + const untrackedChanged = await hasUntrackedFilesChanged( + state, + options.crawlDirectory, + ); + if (!untrackedChanged) { + return applyMaxFilesLimit(state.fileList, options.maxFiles); + } + } + } + } + + const gitResult = await crawlWithGitLsFiles( + stateKey, + options.crawlDirectory, + options.cwd, + options, ); + if (gitResult.success) { + const results = applyMaxFilesLimit(gitResult.files, options.maxFiles); + + if (options.cache) { + const cacheKey = cache.getCacheKey( + options.crawlDirectory, + options.ignore.getFingerprint(), + options.maxDepth, + options.maxFiles, + ); + cache.write(cacheKey, results, options.cacheTtl * 1000); + } + + return results; + } + + if (!gitResult.isGitRepo) { + const rgResult = await crawlWithRipgrep( + stateKey, + options.crawlDirectory, + options.cwd, + options, + ); + if (rgResult.success) { + const results = applyMaxFilesLimit(rgResult.files, options.maxFiles); + + if (options.cache) { + const cacheKey = cache.getCacheKey( + options.crawlDirectory, + options.ignore.getFingerprint(), + options.maxDepth, + options.maxFiles, + ); + cache.write(cacheKey, results, options.cacheTtl * 1000); + } + + return results; + } + } + + const fdirResults = await crawlWithFdir(options); + updateChangeState(stateKey, options.crawlDirectory, fdirResults); + recordRebuild(stateKey); + const limitedResults = applyMaxFilesLimit(fdirResults, options.maxFiles); if (options.cache) { const cacheKey = cache.getCacheKey( @@ -94,8 +783,15 @@ export async function crawl(options: CrawlOptions): Promise { options.maxDepth, options.maxFiles, ); - cache.write(cacheKey, relativeToCwdResults, options.cacheTtl * 1000); + cache.write(cacheKey, limitedResults, options.cacheTtl * 1000); } - return relativeToCwdResults; + return limitedResults; +} + +function applyMaxFilesLimit(results: string[], maxFiles?: number): string[] { + if (maxFiles !== undefined && results.length > maxFiles) { + return results.slice(0, maxFiles); + } + return results; }