Skip to content

perf(server): a checkpoint capture starts 8 git processes instead of 10 #21315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
josephv123 wants to merge 1 commit into
base: main
Choose a base branch
from
+174 −11
Open

perf(server): a checkpoint capture starts 8 git processes instead of 10 #21315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 119 additions & 0 deletions apps/server/src/vcs/GitVcsDriver.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { ChildProcessSpawner } from "effect/process";
import { assert, it } from "@effect/vitest";

import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
import * as HostProcess from "@t3tools/shared/HostProcess";
import * as ServerConfig from "../config.ts";
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
import * as ProcessRunner from "../processRunner.ts";
Expand Down Expand Up @@ -1036,6 +1037,124 @@ it.effect.each(["missing", "invalid"] as const)(
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

const captureRecordingGitCommands = Effect.fn("test.captureRecordingGitCommands")(function* (
cwd: string,
checkpointRef: CheckpointRef,
) {
const liveProcess = yield* VcsProcess.VcsProcess;
const commands: Array<string | undefined> = [];
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
Effect.provideService(VcsProcess.VcsProcess, {
run: (input) => {
let index = 0;
while (input.args[index] === "-C" || input.args[index] === "-c") index += 2;
commands.push(input.args[index]);
return liveProcess.run(input);
},
}),
);
yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
return commands;
});

it.effect.each(["committed", "unborn"] as const)(
"checkpoint capture reads the repository with one rev-parse (HEAD=%s)",
(head) =>
Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const driver = yield* GitVcsDriver.makeVcsDriverShape();
const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-processes-" });
const git = (args: ReadonlyArray<string>) =>
driver.execute({ operation: "checkpoint-test", cwd, args });
const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/test");
if (head === "committed") {
yield* makeCheckpointFixture(driver, cwd);
} else {
yield* git(["init"]);
yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "unstaged\n");
}

const commands = yield* captureRecordingGitCommands(cwd, checkpointRef);

assert.deepStrictEqual(commands, [
"rev-parse",
"config",
...(head === "committed" ? ["read-tree", "ls-files"] : []),
"add",
"write-tree",
"commit-tree",
"update-ref",
]);
assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n");
const [, ...parents] = (yield* git([
"rev-list",
"--parents",
"-n",
"1",
checkpointRef,
])).stdout
.trim()
.split(" ");
assert.deepStrictEqual(
parents,
head === "committed" ? [(yield* git(["rev-parse", "HEAD"])).stdout.trim()] : [],
);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

it.effect("checkpoint capture outside a repository fails with the common dir lookup error", () =>
Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-no-repo-" });

const error = yield* captureRecordingGitCommands(
cwd,
CheckpointRef.make("refs/t3/checkpoints/test"),
).pipe(Effect.flip);

assert.instanceOf(error, VcsProcessExitError);
assert.strictEqual(error.operation, "GitVcsDriver.checkpoints.resolveGitCommonDir");
assert.strictEqual(error.exitCode, 128);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
"checkpoint capture handles a repository path containing a newline",
() =>
Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const driver = yield* GitVcsDriver.makeVcsDriverShape();
const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-new\nline-" });
const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
const originalIndex = yield* fileSystem.readFile(path.join(cwd, ".git", "index"));

const commands = yield* captureRecordingGitCommands(cwd, checkpointRef);

assert.deepStrictEqual(commands, [
"rev-parse",
"rev-parse",
"config",
"rev-parse",
"read-tree",
"ls-files",
"add",
"write-tree",
"commit-tree",
"update-ref",
]);
assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n");
assert.deepEqual(yield* fileSystem.readFile(path.join(cwd, ".git", "index")), originalIndex);
assert.deepStrictEqual(
(yield* fileSystem.readDirectory(path.join(cwd, ".git"))).filter((name) =>
name.startsWith("t3-checkpoint-index-"),
),
[],
);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

it.effect("restores empty checkpoints without changing paths outside the workspace", () =>
Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
Expand Down
66 changes: 55 additions & 11 deletions apps/server/src/vcs/GitVcsDriver.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,15 +769,58 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
}),
);

const resolveFromCwd = (cwd: string, output: string) => {
const resolved = output.trim();
return path.isAbsolute(resolved) ? resolved : path.resolve(cwd, resolved);
};

const resolveGitCommonDir = (cwd: string) =>
Effect.gen(function* () {
const result = yield* execute({
operation: "GitVcsDriver.checkpoints.resolveGitCommonDir",
cwd,
args: ["rev-parse", "--git-common-dir"],
});
const gitCommonDir = result.stdout.trim();
return path.isAbsolute(gitCommonDir) ? gitCommonDir : path.resolve(cwd, gitCommonDir);
return resolveFromCwd(cwd, result.stdout);
});

// --git-common-dir precedes --path-format, so it prints in its default form. --verify prints
// HEAD's oid after both paths, and with --quiet an unborn HEAD exits 1 with only the paths.
const resolveCaptureRepository = (cwd: string) =>
Effect.gen(function* () {
const result = yield* execute({
operation: "GitVcsDriver.checkpoints.resolveCaptureRepository",
cwd,
args: [
"rev-parse",
"--git-common-dir",
"--path-format=absolute",
"--git-path",
"index",
"--verify",
"--quiet",
"HEAD",
],
allowNonZeroExit: true,
});
const headExists = result.exitCode === 0;
const headChecked = headExists || result.exitCode === 1;
const lines = result.stdout.split("\n");
const [gitCommonDir = "", indexPath = ""] = lines;
if (headChecked && lines.length === (headExists ? 4 : 3) && lines.at(-1) === "") {
return {
gitCommonDir: resolveFromCwd(cwd, gitCommonDir),
indexPath: indexPath.trim(),
headExists,
};
}
// Git before 2.31 echoes --path-format, and a path containing a newline spans lines.
// Separate commands read the paths as before and report a failure the same way.
return {
gitCommonDir: yield* resolveGitCommonDir(cwd),
indexPath: undefined,
headExists: headChecked ? headExists : yield* hasHeadCommit(cwd),
};
});

// Git renames loose objects and refs into place without fsync by default, so
Expand Down Expand Up @@ -806,7 +849,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
"-c",
"splitIndex.sharedIndexExpire=never",
];
const gitCommonDir = yield* resolveGitCommonDir(input.cwd);
const { gitCommonDir, indexPath, headExists } = yield* resolveCaptureRepository(input.cwd);
const indexId = yield* crypto.randomUUIDv4().pipe(Effect.orDie);
const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`);
const commitEnv: NodeJS.ProcessEnv = {
Expand All @@ -825,7 +868,6 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
);

yield* Effect.gen(function* () {
const headExists = yield* hasHeadCommit(input.cwd);
const sparseConfig = yield* execute({
operation,
cwd: input.cwd,
Expand All @@ -844,17 +886,19 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
}
if (headExists) {
const reusedIndex = yield* Effect.gen(function* () {
const indexPath = yield* execute({
operation,
cwd: input.cwd,
args: ["rev-parse", "--path-format=absolute", "--git-path", "index"],
});
const { mtime } = yield* fileSystem.stat(indexPath.stdout.trim());
const userIndexPath =
indexPath ??
(yield* execute({
operation,
cwd: input.cwd,
args: ["rev-parse", "--path-format=absolute", "--git-path", "index"],
})).stdout.trim();
const { mtime } = yield* fileSystem.stat(userIndexPath);
if (Option.isNone(mtime)) return false;
// Stay below the source timestamp even if Date rounded up, preserving Git's racy check.
const indexTime = Math.floor((mtime.value.getTime() - 1) / 1000);
if (indexTime <= 0) return false;
yield* fileSystem.copyFile(indexPath.stdout.trim(), tempIndexPath);
yield* fileSystem.copyFile(userIndexPath, tempIndexPath);
yield* execute({
operation,
cwd: input.cwd,
Expand Down
Loading