fix(server): avoid slow checkpoint index rebuilds

apps/server/src/vcs/GitVcsDriver.test.ts

@@ -1,3 +1,6 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFS from "node:fs";
+
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as Effect from "effect/Effect";
 import * as FileSystem from "effect/FileSystem";
@@ -7,7 +10,7 @@ import * as PlatformError from "effect/PlatformError";
 import { ChildProcessSpawner } from "effect/unstable/process";
 import { assert, it } from "@effect/vitest";
 
-import { GitCommandError } from "@t3tools/contracts";
+import { CheckpointRef, GitCommandError } from "@t3tools/contracts";
 import * as ServerConfig from "../config.ts";
 import * as GitVcsDriver from "./GitVcsDriver.ts";
 import * as VcsProcess from "./VcsProcess.ts";
@@ -108,3 +111,113 @@ it.effect("GitVcsDriver forwards execute env to the VCS process", () => {
     ),
   );
 });
+
+it.effect("GitVcsDriver normalizes checkpoint temp indexes and removes lock files", () => {
+  let gitDir = "";
+  let currentIndexPath = "";
+  let tempIndexPath = "";
+  let readTreeArgs: ReadonlyArray<string> | undefined;
+  let updateIndexArgs: ReadonlyArray<string> | undefined;
+  let copiedIndexContents = "";
+
+  const processLayer = Layer.mock(VcsProcess.VcsProcess)({
+    run: (input) =>
+      Effect.sync(() => {
+        const args = input.args.slice(2);
+        let stdout = "";
+
+        if (args[0] === "rev-parse" && args[1] === "--git-common-dir") {
+          stdout = gitDir;
+        } else if (args[0] === "rev-parse" && args[1] === "--git-path") {
+          stdout = currentIndexPath;
+        } else if (args[0] === "read-tree") {
+          readTreeArgs = args;
+        } else if (args[0] === "ls-files") {
+          stdout = "H tracked.txt\0";
+        } else if (args[0] === "update-index") {
+          updateIndexArgs = args;
+        } else if (args[0] === "add") {
+          tempIndexPath = input.env?.GIT_INDEX_FILE ?? "";
+          copiedIndexContents = NodeFS.readFileSync(tempIndexPath, "utf8");
+          NodeFS.writeFileSync(`${tempIndexPath}.lock`, "locked");
+        } else if (args[0] === "write-tree") {
+          stdout = "tree-oid\n";
+        } else if (args[0] === "commit-tree") {
+          stdout = "commit-oid\n";
+        }
+
+        return {
+          exitCode: ChildProcessSpawner.ExitCode(0),
+          stdout,
+          stderr: "",
+          stdoutTruncated: false,
+          stderrTruncated: false,
+        };
+      }),
+  });
+
+  return Effect.gen(function* () {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-index-" });
+    gitDir = path.join(cwd, ".git");
+    currentIndexPath = path.join(gitDir, "index");
+    yield* fileSystem.makeDirectory(gitDir, { recursive: true });
+    yield* fileSystem.writeFileString(currentIndexPath, "current-index");
+
+    const driver = yield* GitVcsDriver.makeVcsDriverShape();
+    yield* driver.checkpoints.captureCheckpoint({
+      cwd,
+      checkpointRef: CheckpointRef.make("refs/t3/checkpoints/test"),
+    });
+
+    assert.deepStrictEqual(readTreeArgs, ["read-tree", "--reset", "HEAD"]);
+    assert.deepStrictEqual(updateIndexArgs, [
+      "update-index",
+      "--no-split-index",
+      "--no-untracked-cache",
+      "--no-fsmonitor",
+    ]);
+    assert.strictEqual(copiedIndexContents, "current-index");
+    assert.strictEqual(NodeFS.existsSync(tempIndexPath), false);
+    assert.strictEqual(NodeFS.existsSync(`${tempIndexPath}.lock`), false);
+  }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, processLayer)));
+});
+
+it.effect("GitVcsDriver checkpoints files hidden by worktree index flags", () =>
+  Effect.gen(function* () {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-flags-" });
+    const driver = yield* GitVcsDriver.makeVcsDriverShape();
+    const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/index-flags");
+
+    yield* runGit(cwd, ["init"]);
+    yield* runGit(cwd, ["config", "user.email", "test@test.com"]);
+    yield* runGit(cwd, ["config", "user.name", "Test"]);
+    yield* fileSystem.writeFileString(path.join(cwd, "assumed.txt"), "committed\n");
+    yield* fileSystem.writeFileString(path.join(cwd, "skipped.txt"), "committed\n");
+    yield* runGit(cwd, ["add", "assumed.txt", "skipped.txt"]);
+    yield* runGit(cwd, ["commit", "-m", "initial"]);
+    yield* runGit(cwd, ["update-index", "--assume-unchanged", "assumed.txt"]);
+    yield* runGit(cwd, ["update-index", "--skip-worktree", "skipped.txt"]);
+    yield* fileSystem.writeFileString(path.join(cwd, "assumed.txt"), "checkpointed\n");
+    yield* fileSystem.writeFileString(path.join(cwd, "skipped.txt"), "checkpointed\n");
+
+    yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
+
+    const assumed = yield* driver.execute({
+      operation: "GitVcsDriver.test.readAssumedCheckpointFile",
+      cwd,
+      args: ["show", `${checkpointRef}:assumed.txt`],
+    });
+    const skipped = yield* driver.execute({
+      operation: "GitVcsDriver.test.readSkippedCheckpointFile",
+      cwd,
+      args: ["show", `${checkpointRef}:skipped.txt`],
+    });
+
+    assert.strictEqual(assumed.stdout, "checkpointed\n");
+    assert.strictEqual(skipped.stdout, "checkpointed\n");
+  }).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
+);

apps/server/src/vcs/GitVcsDriver.ts

@@ -331,6 +331,7 @@ export class GitVcsDriver extends Context.Service<
 const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
 const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
 const CHECKPOINT_DIFF_MAX_OUTPUT_BYTES = 10_000_000;
+const CHECKPOINT_INDEX_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
 const WORKSPACE_GIT_HARDENED_CONFIG_ARGS = [
   "-c",
   "core.fsmonitor=false",
@@ -708,6 +709,17 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
       return path.isAbsolute(gitCommonDir) ? gitCommonDir : path.resolve(cwd, gitCommonDir);
     });
 
+  const resolveGitIndexPath = (cwd: string) =>
+    Effect.gen(function* () {
+      const result = yield* execute({
+        operation: "GitVcsDriver.checkpoints.resolveGitIndexPath",
+        cwd,
+        args: ["rev-parse", "--git-path", "index"],
+      });
+      const indexPath = result.stdout.trim();
+      return path.isAbsolute(indexPath) ? indexPath : path.resolve(cwd, indexPath);
+    });
+
   const checkpoints: VcsDriver.VcsCheckpointOps = {
     captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
       const operation = "GitVcsDriver.checkpoints.captureCheckpoint";
@@ -727,17 +739,104 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
 
       const cleanupTempIndex = fileSystem
         .remove(tempIndexPath, { force: true })
-        .pipe(Effect.ignore);
+        .pipe(
+          Effect.ignore,
+          Effect.andThen(
+            fileSystem.remove(`${tempIndexPath}.lock`, { force: true }).pipe(Effect.ignore),
+          ),
+        );
 
       yield* Effect.gen(function* () {
         const headExists = yield* hasHeadCommit(input.cwd);
         if (headExists) {
-          yield* execute({
-            operation,
-            cwd: input.cwd,
-            args: ["read-tree", "HEAD"],
-            env: commitEnv,
-          });
+          const currentIndexPath = yield* resolveGitIndexPath(input.cwd);
+          const copiedCurrentIndex = yield* fileSystem.exists(currentIndexPath).pipe(
+            Effect.flatMap((exists) =>
+              exists
+                ? fileSystem.copyFile(currentIndexPath, tempIndexPath).pipe(Effect.as(true))
+                : Effect.succeed(false),
+            ),
+            Effect.orElseSucceed(() => false),
+          );
+          if (copiedCurrentIndex) {
+            // Start from HEAD while retaining the copied index's stat cache.
+            // The remaining commands clear flags and extensions that can make
+            // `git add` trust stale worktree state.
+            yield* execute({
+              operation,
+              cwd: input.cwd,
+              args: ["read-tree", "--reset", "HEAD"],
+              env: commitEnv,
+            });
+            const indexEntries = yield* execute({
+              operation,
+              cwd: input.cwd,
+              args: ["ls-files", "-v", "-z"],
+              env: commitEnv,
+              maxOutputBytes: CHECKPOINT_INDEX_MAX_OUTPUT_BYTES,
+            });
+            if (indexEntries.stdoutTruncated) {
+              yield* execute({
+                operation,
+                cwd: input.cwd,
+                args: ["read-tree", "--empty"],
+                env: commitEnv,
+              });
+              yield* execute({
+                operation,
+                cwd: input.cwd,
+                args: ["read-tree", "HEAD"],
+                env: commitEnv,
+              });
+            } else {
+              const indexRecords = indexEntries.stdout.split("\0");
+              const assumeUnchangedPaths = indexRecords
+                .filter((entry) => {
+                  const tag = entry.charAt(0);
+                  return tag >= "a" && tag <= "z";
+                })
+                .map((entry) => entry.slice(2));
+              if (assumeUnchangedPaths.length > 0) {
+                yield* execute({
+                  operation,
+                  cwd: input.cwd,
+                  args: ["update-index", "--no-assume-unchanged", "-z", "--stdin"],
+                  stdin: `${assumeUnchangedPaths.join("\0")}\0`,
+                  env: commitEnv,
+                });
+              }
+              const skipWorktreePaths = indexRecords
+                .filter((entry) => entry.charAt(0).toUpperCase() === "S")
+                .map((entry) => entry.slice(2));
+              if (skipWorktreePaths.length > 0) {
+                yield* execute({
+                  operation,
+                  cwd: input.cwd,
+                  args: ["update-index", "--no-skip-worktree", "-z", "--stdin"],
+                  stdin: `${skipWorktreePaths.join("\0")}\0`,
+                  env: commitEnv,
+                });
+              }
+              yield* execute({
+                operation,
+                cwd: input.cwd,
+                args: [
+                  "update-index",
+                  "--no-split-index",
+                  "--no-untracked-cache",
+                  "--no-fsmonitor",
+                ],
+                env: commitEnv,
+              });
+            }
+          } else {
+            yield* execute({
+              operation,
+              cwd: input.cwd,
+              args: ["read-tree", "HEAD"],
+              env: commitEnv,
+            });
+          }
         }
 
         yield* execute({