diff --git a/packages/core/src/plugin/vcs/git.ts b/packages/core/src/plugin/vcs/git.ts index dd5fc2b1657b..9efdc74d2982 100644 --- a/packages/core/src/plugin/vcs/git.ts +++ b/packages/core/src/plugin/vcs/git.ts @@ -1,5 +1,9 @@ export * as VcsGitPlugin from "./git.js" +import { randomUUID } from "crypto" +import fs from "fs/promises" +import os from "os" +import path from "path" import { define } from "@opencode/plugin/effect/plugin" import { Effect } from "effect" import { ChildProcess } from "effect/unstable/process" @@ -16,6 +20,7 @@ import { MAX_PATCH_BYTES, MAX_TOTAL_PATCH_BYTES, PATCH_CONTEXT_LINES, + splitGitPatch, } from "../../vcs/patch.js" import type { Patch } from "../../vcs/patch.js" @@ -61,7 +66,7 @@ export const Plugin = define({ function make(proc: AppProcess.Interface, input: { directory: string; worktree: string }) { // Listing commands scope pathspecs to the requested directory; per-file // commands run from the worktree root because git lists root-relative paths. - const ctx: Ctx = { git: makeGit(proc), directory: input.directory, worktree: input.worktree } + const ctx: Ctx = { proc, git: makeGit(proc), directory: input.directory, worktree: input.worktree } return { info: Effect.fn("VcsGit.info")(function* () { @@ -82,13 +87,19 @@ function make(proc: AppProcess.Interface, input: { directory: string; worktree: { concurrency: 2 }, ) const map = nums(stats) + const untracked = yield* untrackedBatch( + ctx, + list.filter((item) => item.status === "added" && !map.has(item.file)).map((item) => item.file), + ) return yield* Effect.forEach( list.toSorted((a, b) => a.file.localeCompare(b.file)), (item) => Effect.gen(function* () { const stat = map.get(item.file) ?? - (item.status === "added" ? yield* git.statUntracked(ctx.worktree, item.file) : undefined) + (item.status === "added" + ? (untracked.stats.get(item.file) ?? (yield* git.statUntracked(ctx.worktree, item.file))) + : undefined) return { file: item.file, additions: stat?.additions ?? 0, @@ -142,6 +153,7 @@ interface PatchOptions extends GitDiffOptions { } interface Ctx { + readonly proc: AppProcess.Interface readonly git: GitOps readonly directory: string readonly worktree: string @@ -173,12 +185,33 @@ const kind = (code: string): Kind => { const nuls = (text: string) => text.split("\0").filter(Boolean) -function makeGit(proc: AppProcess.Interface) { +const numstat = (text: string) => + nuls(text).flatMap((item) => { + const a = item.indexOf("\t") + const b = item.indexOf("\t", a + 1) + if (a === -1 || b === -1) return [] + const file = item.slice(b + 1) + if (!file) return [] + const adds = item.slice(0, a) + const dels = item.slice(a + 1, b) + const additions = adds === "-" ? 0 : Number.parseInt(adds || "0", 10) + const deletions = dels === "-" ? 0 : Number.parseInt(dels || "0", 10) + return [ + { + file, + additions: Number.isFinite(additions) ? additions : 0, + deletions: Number.isFinite(deletions) ? deletions : 0, + } satisfies Stat, + ] + }) + +function makeGit(proc: AppProcess.Interface, env?: Record) { const run = Effect.fnUntraced( function* (args: string[], opts: { cwd: string; maxOutputBytes?: number }) { const result = yield* proc.run( ChildProcess.make(gitExecutable, [...cfg, ...args], { cwd: opts.cwd, + env, extendEnv: true, stdin: "ignore", }), @@ -357,29 +390,12 @@ function makeGit(proc: AppProcess.Interface) { }) const stats = Effect.fn("VcsGit.stats")(function* (cwd: string, ref: string, target?: string) { - return nuls( + return numstat( yield* text( ["diff", "--no-ext-diff", "--no-renames", "--numstat", "-z", ref, ...(target ? [target] : []), "--", "."], { cwd }, ), - ).flatMap((item) => { - const a = item.indexOf("\t") - const b = item.indexOf("\t", a + 1) - if (a === -1 || b === -1) return [] - const file = item.slice(b + 1) - if (!file) return [] - const adds = item.slice(0, a) - const dels = item.slice(a + 1, b) - const additions = adds === "-" ? 0 : Number.parseInt(adds || "0", 10) - const deletions = dels === "-" ? 0 : Number.parseInt(dels || "0", 10) - return [ - { - file, - additions: Number.isFinite(additions) ? additions : 0, - deletions: Number.isFinite(deletions) ? deletions : 0, - } satisfies Stat, - ] - }) + ) }) const patch = Effect.fn("VcsGit.patch")(function* (cwd: string, ref: string, file: string, options?: PatchOptions) { @@ -459,6 +475,37 @@ function makeGit(proc: AppProcess.Interface) { } satisfies Stat }) + // Paths are literal so names containing pathspec magic such as `*` or `[` match only themselves. + const intentToAdd = Effect.fn("VcsGit.intentToAdd")(function* (cwd: string, list: string) { + yield* run( + [ + "--literal-pathspecs", + "-c", + "core.splitIndex=false", + "add", + "--intent-to-add", + "--force", + "--sparse", + "--ignore-errors", + `--pathspec-from-file=${list}`, + "--pathspec-file-nul", + ], + { cwd }, + ) + }) + + const worktreeStats = Effect.fn("VcsGit.worktreeStats")(function* (cwd: string) { + return numstat(yield* text(["diff", "--no-ext-diff", "--no-renames", "--numstat", "-z"], { cwd })) + }) + + const worktreePatch = Effect.fn("VcsGit.worktreePatch")(function* (cwd: string, options?: PatchOptions) { + const result = yield* run( + ["diff", "--patch", "--no-ext-diff", "--no-renames", `--unified=${options?.context ?? 3}`], + { cwd, maxOutputBytes: options?.maxOutputBytes }, + ) + return { text: result.text(), truncated: result.truncated } satisfies Patch + }) + return { branch, branches, @@ -473,9 +520,88 @@ function makeGit(proc: AppProcess.Interface) { patchAll, patchUntracked, statUntracked, + intentToAdd, + worktreeStats, + worktreePatch, } } +const emptyUntracked = () => ({ stats: new Map(), patches: new Map() }) + +/** + * Prefetches what `statUntracked` and `patchUntracked` report, in three Git processes instead of two + * per file. The files are recorded intent-to-add in an otherwise empty throwaway index, so a plain + * `git diff` against it shows exactly those files as new, with the same attribute handling and patch + * text as `git diff --no-index /dev/null `. Files the batch cannot reproduce byte for byte are + * left out, and callers fall back to the per-file commands. + */ +const untrackedBatch = Effect.fnUntraced(function* ( + ctx: Ctx, + files: readonly string[], + // Patches are only built when requested; status needs line counts alone. + patches?: DiffOptions, +) { + const eligible = yield* Effect.promise(() => + Promise.all( + files.map(async (file) => { + // Directories (nested repositories) and symlinks diff differently through an index. + if (file.endsWith("/")) return + const stat = await fs.lstat(path.join(ctx.worktree, file)).catch(() => undefined) + if (!stat?.isFile()) return + // A file this large could crowd every other file out of the batch's output budget. + if (stat.size > MAX_TOTAL_PATCH_BYTES / 2) return + return file + }), + ).then((list) => list.filter((file) => file !== undefined)), + ) + // The batch costs up to three processes; for a single file the per-file commands are cheaper. + if (eligible.length < 2) return emptyUntracked() + + return yield* Effect.acquireUseRelease( + Effect.promise(async () => { + const base = path.join(os.tmpdir(), `opencode-untracked-${randomUUID()}`) + await fs.writeFile(`${base}.list`, eligible.map((file) => `${file}\0`).join("")) + return base + }), + (base) => + Effect.gen(function* () { + const git = makeGit(ctx.proc, { GIT_INDEX_FILE: `${base}.index` }) + yield* git.intentToAdd(ctx.worktree, `${base}.list`) + const [stats, patch] = yield* Effect.all( + [ + git.worktreeStats(ctx.worktree), + patches + ? git.worktreePatch(ctx.worktree, { + context: patches.context ?? PATCH_CONTEXT_LINES, + maxOutputBytes: MAX_TOTAL_PATCH_BYTES, + }) + : Effect.succeed({ text: "", truncated: false } satisfies Patch), + ], + { concurrency: 2 }, + ) + const wanted = new Set(eligible) + const headers = new Map(eligible.map((file) => [`diff --git a/${file} b/${file}`, file])) + return { + stats: new Map(stats.filter((item) => wanted.has(item.file)).map((item) => [item.file, item] as const)), + // Only chunks with the default `a/` and `b/` prefixes match the per-file output; configured + // prefixes and quoted paths fall back to it. + patches: new Map( + splitGitPatch(patch).flatMap((chunk) => { + const file = headers.get(chunk.slice(0, chunk.indexOf("\n"))) + return file === undefined ? [] : [[file, chunk] as const] + }), + ), + } + }), + (base) => + Effect.promise(() => + Promise.all( + [`${base}.list`, `${base}.index`, `${base}.index.lock`].map((file) => fs.rm(file, { force: true })), + ), + ), + ) +}) + const nums = (list: Stat[]) => new Map(list.map((item) => [item.file, { additions: item.additions, deletions: item.deletions }] as const)) @@ -536,6 +662,7 @@ const patchForItem = Effect.fnUntraced(function* ( ref: string | undefined, item: Item, batch: { patches: Map; capped: boolean }, + untracked: Map, capped: boolean, options?: GitDiffOptions, ) { @@ -544,6 +671,8 @@ const patchForItem = Effect.fnUntraced(function* ( const batched = batch.patches.get(item.file) if (batched !== undefined) return batched if (item.code !== "??" && batch.capped) return emptyPatch(item.file) + const prefetched = item.code === "??" || !ref ? untracked.get(item.file) : undefined + if (prefetched !== undefined) return prefetched return yield* nativePatch(ctx, ref, item, options) }) @@ -555,6 +684,15 @@ const files = Effect.fnUntraced(function* ( batch: { patches: Map; capped: boolean }, options?: GitDiffOptions, ) { + const untracked = yield* untrackedBatch( + ctx, + list + .filter( + (item) => item.code === "??" || !ref || (!options?.target && item.status === "added" && !map.has(item.file)), + ) + .map((item) => item.file), + options ?? {}, + ) const next: FileDiff.Info[] = [] let total = 0 let capped = false @@ -562,8 +700,10 @@ const files = Effect.fnUntraced(function* ( for (const item of list.toSorted((a, b) => a.file.localeCompare(b.file))) { const stat = map.get(item.file) ?? - (!options?.target && item.status === "added" ? yield* ctx.git.statUntracked(ctx.worktree, item.file) : undefined) - const patch = yield* patchForItem(ctx, ref, item, batch, capped, options) + (!options?.target && item.status === "added" + ? (untracked.stats.get(item.file) ?? (yield* ctx.git.statUntracked(ctx.worktree, item.file))) + : undefined) + const patch = yield* patchForItem(ctx, ref, item, batch, untracked.patches, capped, options) const result: { patch: string; capped: boolean } = capped ? { patch, capped: true } : totalPatch(item.file, patch, total) diff --git a/packages/core/test/vcs.test.ts b/packages/core/test/vcs.test.ts index 2f70991e2f66..8bcc471adfd3 100644 --- a/packages/core/test/vcs.test.ts +++ b/packages/core/test/vcs.test.ts @@ -581,6 +581,65 @@ describe("Vcs", () => { ), ) + it.live("batches untracked files without spawning Git per file or touching the index", () => + withTmp((directory) => + Effect.gen(function* () { + const count = 40 + yield* Effect.promise(async () => { + await initRepo(directory) + await fs.writeFile(path.join(directory, "keep.txt"), "one\n") + await fs.writeFile(path.join(directory, ".gitignore"), "*.log\n") + await commitAll(directory, "initial") + await fs.writeFile(path.join(directory, "staged.txt"), "staged\n") + await $`git add staged.txt`.cwd(directory).quiet() + await fs.mkdir(path.join(directory, "evals/nested"), { recursive: true }) + await Promise.all( + Array.from({ length: count }, (_, index) => + fs.writeFile(path.join(directory, `evals/nested/file-${index}.md`), `line ${index}\nmore\n`), + ), + ) + await fs.writeFile(path.join(directory, "evals/ignored.log"), "ignored\n") + }) + const index = yield* Effect.promise(() => fs.readFile(path.join(directory, ".git/index"))) + const vcs = yield* Vcs.Service + const processes = yield* AppProcess.Service + let spawned = 0 + const context = host() + yield* VcsGitPlugin.Plugin.effect({ + ...context, + vcs: { ...context.vcs, transform: vcs.transform, reload: vcs.reload }, + }).pipe( + Effect.provideService( + AppProcess.Service, + AppProcess.Service.of({ + ...processes, + run: (command, options) => { + spawned++ + return processes.run(command, options) + }, + }), + ), + ) + + const diff = yield* vcs.diff("working") + const status = yield* vcs.status() + expect(spawned).toBeLessThan(count) + expect(diff.map((row) => row.file)).toEqual(status.map((row) => row.file)) + expect(diff).toHaveLength(count + 1) + expect(diff.some((row) => row.file.endsWith(".log"))).toBeFalse() + expect(diff.find((row) => row.file === "staged.txt")).toMatchObject({ status: "added", additions: 1 }) + expect(diff.find((row) => row.file === "evals/nested/file-7.md")).toMatchObject({ + status: "added", + additions: 2, + deletions: 0, + }) + expect(diff.find((row) => row.file === "evals/nested/file-7.md")?.patch).toContain("+line 7") + expect(status.find((row) => row.file === "evals/nested/file-7.md")).toMatchObject({ additions: 2 }) + expect(yield* Effect.promise(() => fs.readFile(path.join(directory, ".git/index")))).toEqual(index) + }).pipe(provide(directory, { git: true, worktree: directory })), + ), + ) + it.live("caches branch info and publishes HEAD changes", () => withGit((directory) => Effect.gen(function* () {