feat(storage): clean up settled thread worktrees

apps/server/src/environment/ServerEnvironment.ts

@@ -229,6 +229,7 @@ export const make = Effect.gen(function* () {
       threadAutoSettlement: true,
       storageCleanup: true,
       projectWorktreeCleanup: true,
+      settledWorktreeCleanup: true,
       threadRestartContinuation: true,
       projectSettingsOverrides: true,
       threadSnooze: true,

apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts

@@ -16,6 +16,7 @@ import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
 import * as Path from "effect/Path";
 import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
+import { GitWorkflowService } from "../git/GitWorkflowService.ts";
 import {
   CheckpointRollbackServiceV2,
   layer as rollbackLayer,
@@ -106,6 +107,7 @@ it.effect.each([
         Layer.mergeAll(
           NodeServices.layer,
           idAllocatorLayer,
+          Layer.mock(GitWorkflowService)({}),
           Layer.mock(ProjectStore.ProjectStoreV2)({
             get: () => Effect.succeed(Option.some({ workspaceRoot: parent } as never)),
           }),

apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts

@@ -4,18 +4,24 @@ import {
   CheckpointId,
   CheckpointScopeId,
   type OrchestrationV2ThreadProjection,
+  ProjectId,
   ProviderInstanceId,
   ProviderSessionId,
   ProviderThreadId,
   ThreadId,
 } from "@t3tools/contracts";
 import * as Effect from "effect/Effect";
+import * as Deferred from "effect/Deferred";
+import * as Fiber from "effect/Fiber";
 import * as FileSystem from "effect/FileSystem";
 import * as Path from "effect/Path";
 import * as Option from "effect/Option";
 import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
 import * as Layer from "effect/Layer";
 
+import * as GitWorkflow from "../git/GitWorkflowService.ts";
+import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
+import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
 import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
 import { isCheckpointRestoreIsolated } from "./CheckpointRestoreSafety.ts";
 import * as CheckpointService from "./CheckpointService.ts";
@@ -36,6 +42,7 @@ const checkpointRollbackServiceLayer = CheckpointRollbackService.layer.pipe(
   Layer.provide(
     Layer.mergeAll(
       NodeServices.layer,
+      Layer.mock(GitWorkflow.GitWorkflowService)({}),
       Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeed(unrelatedProject) }),
     ),
   ),
@@ -263,7 +270,7 @@ it.effect("rejects a rollback when provider selection changed before execution",
   }).pipe(Effect.provide(testLayer));
 });
 
-it.effect("reports a missing provider turn as a structured rollback failure", () => {
+it.effect("rejects an unavailable rollback target before opening a session", () => {
   const threadId = ThreadId.make("thread:rollback-provider-turn-unavailable");
   const providerThreadId = ProviderThreadId.make(
     "provider-thread:rollback-provider-turn-unavailable",
@@ -275,9 +282,11 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
   const scopeId = CheckpointScopeId.make("checkpoint-scope:rollback-provider-turn-unavailable");
   const providerInstanceId = ProviderInstanceId.make("provider_rollback_provider_turn_unavailable");
   const restore = vi.fn(() => Effect.die("checkpoint restore must not run"));
+  const open = vi.fn(() => Effect.die("provider session open must not run"));
   const projection = {
     thread: {
-      worktreePath: process.cwd(),
+      worktreePath: "/tmp/missing-invalid-rollback-worktree",
+      branch: "feature/rollback",
       activeProviderThreadId: providerThreadId,
       modelSelection: { instanceId: providerInstanceId, model: "test-model" },
     },
@@ -306,7 +315,7 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
             }),
         }),
         Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
-          open: () => Effect.succeed({} as never),
+          open,
         }),
         Layer.mock(RuntimePolicy.RuntimePolicyV2)({
           resolve: () => Effect.succeed({} as never),
@@ -318,14 +327,9 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
   return Effect.gen(function* () {
     const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
     const error = yield* service
-      .execute({
-        threadId,
-        providerThreadId,
-        checkpointId,
-        scopeId,
-      })
+      .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles: false })
       .pipe(Effect.flip);
-
+    assert.equal(open.mock.calls.length, 0);
     assert.equal(error.reason, "provider-turn-unavailable");
     assert.equal(
       error.message,
@@ -336,6 +340,162 @@ it.effect("reports a missing provider turn as a structured rollback failure", ()
   }).pipe(Effect.provide(testLayer));
 });
 
+it.effect.each([
+  { order: "normal", restoreFiles: true },
+  { order: "cleanup-first", restoreFiles: true },
+  { order: "cleanup-first", restoreFiles: false },
+  { order: "rollback-first", restoreFiles: true },
+])("locks rollback with %s", ({ order, restoreFiles }) =>
+  Effect.gen(function* () {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const workspaceRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-rollback-" });
+    const worktreePath = path.join(workspaceRoot, "worktree");
+    yield* fileSystem.makeDirectory(worktreePath);
+    const threadId = ThreadId.make("rollback-worktree");
+    const projectId = ProjectId.make("rollback-worktree-project");
+    const providerThreadId = ProviderThreadId.make("rollback-worktree-provider-thread");
+    const providerSessionId = ProviderSessionId.make("rollback-worktree-provider-session");
+    const providerInstanceId = ProviderInstanceId.make("rollback-worktree-provider-instance");
+    const checkpointId = CheckpointId.make("rollback-worktree-checkpoint");
+    const scopeId = CheckpointScopeId.make("rollback-worktree-scope");
+    const branch = "feature/rollback";
+    const release = Deferred.makeUnsafe<void>();
+    const openEntered = Deferred.makeUnsafe<void>();
+    const calls: string[] = [];
+    let cleanupEntered = false;
+    const providerThread = { id: providerThreadId, providerSessionId, providerInstanceId };
+    const projection = {
+      thread: {
+        id: threadId,
+        projectId,
+        worktreePath,
+        branch,
+        activeProviderThreadId: providerThreadId,
+        modelSelection: { instanceId: providerInstanceId, model: "test-model" },
+      },
+      providerThreads: [providerThread],
+      providerSessions: [],
+      checkpoints: [{ id: checkpointId, scopeId, status: "ready", appRunOrdinal: 0 }],
+      checkpointScopes: [{ id: scopeId, cwd: worktreePath }],
+      runs: [],
+      attempts: [],
+      nodes: [],
+      providerTurns: [],
+    } as unknown as OrchestrationV2ThreadProjection;
+    const testLayer = CheckpointRollbackService.layer.pipe(
+      Layer.provide(
+        Layer.mergeAll(
+          NodeServices.layer,
+          Layer.mock(ProjectStore.ProjectStoreV2)({
+            get: (id) => {
+              assert.equal(id, projectId);
+              return Effect.succeed(Option.some({ workspaceRoot } as never));
+            },
+          }),
+          Layer.mock(GitWorkflow.GitWorkflowService)({
+            pruneWorktrees: (input) =>
+              Effect.sync(() => {
+                assert.deepEqual(input, { cwd: workspaceRoot });
+                calls.push("prune");
+              }),
+            createWorktree: (input) =>
+              Effect.gen(function* () {
+                assert.deepEqual(input, {
+                  cwd: workspaceRoot,
+                  refName: branch,
+                  path: worktreePath,
+                });
+                calls.push("create");
+                yield* fileSystem.makeDirectory(worktreePath).pipe(Effect.orDie);
+                return {} as never;
+              }),
+          }),
+          Layer.mock(CheckpointService.CheckpointServiceV2)({
+            restore: () =>
+              Effect.gen(function* () {
+                assert.isTrue(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie));
+                calls.push("files");
+              }),
+          }),
+          Layer.mock(EventSink.EventSinkV2)({
+            write: () =>
+              Effect.sync(() => {
+                calls.push("projection");
+                return [];
+              }),
+          }),
+          IdAllocator.layer,
+          Layer.mock(ProjectionStore.ProjectionStoreV2)({
+            getThreadRecords: () => Effect.succeed(projection),
+            getShellSnapshot: () =>
+              Effect.succeed({
+                schemaVersion: 1,
+                snapshotSequence: 0,
+                threads: [],
+                archivedThreads: [],
+              }),
+          }),
+          Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
+            open: () =>
+              Effect.gen(function* () {
+                if (!(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie))) {
+                  return yield* new ProviderWorkspaceMissingError({ threadId, cwd: worktreePath });
+                }
+                calls.push("session");
+                yield* Deferred.succeed(openEntered, undefined);
+                if (order === "rollback-first") yield* Deferred.await(release);
+                return {} as never;
+              }),
+          }),
+          Layer.mock(RuntimePolicy.RuntimePolicyV2)({ resolve: () => Effect.succeed({} as never) }),
+        ),
+      ),
+    );
+    yield* Effect.gen(function* () {
+      const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
+      const cleanupEffect = withWorkspaceLease(
+        worktreePath,
+        Effect.gen(function* () {
+          cleanupEntered = true;
+          if (order === "cleanup-first") yield* Deferred.await(release);
+          yield* fileSystem.remove(worktreePath, { recursive: true });
+          calls.push("cleanup");
+        }),
+      );
+      const cleanup =
+        order === "cleanup-first"
+          ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
+          : undefined;
+      const rollback = yield* service
+        .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles })
+        .pipe(Effect.forkChild({ startImmediately: true }));
+      if (order === "cleanup-first") assert.deepEqual(calls, []);
+      if (order === "rollback-first") yield* Deferred.await(openEntered);
+      const laterCleanup =
+        order === "rollback-first"
+          ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
+          : undefined;
+      if (order === "rollback-first") assert.isFalse(cleanupEntered);
+      yield* Deferred.succeed(release, undefined);
+      if (cleanup !== undefined) yield* Fiber.join(cleanup);
+      yield* Fiber.join(rollback);
+      if (laterCleanup !== undefined) yield* Fiber.join(laterCleanup);
+      const rollbackCalls = restoreFiles
+        ? ["session", "files", "projection"]
+        : ["session", "projection"];
+      assert.deepEqual(
+        calls,
+        order === "cleanup-first"
+          ? ["cleanup", "prune", "create", ...rollbackCalls]
+          : order === "rollback-first"
+            ? [...rollbackCalls, "cleanup"]
+            : rollbackCalls,
+      );
+    }).pipe(Effect.provide(testLayer));
+  }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
+);
+
 it.effect.each([
   { restoreFiles: true, shared: "none" },
   { restoreFiles: false, shared: "root" },