feat(storage): clean up settled thread worktrees

apps/server/src/environment/ServerEnvironment.ts

@@ -229,6 +229,7 @@ export const make = Effect.gen(function* () {
       threadAutoSettlement: true,
       storageCleanup: true,
       projectWorktreeCleanup: true,
+      settledWorktreeCleanup: true,
       threadRestartContinuation: true,
       projectSettingsOverrides: true,
       threadSnooze: true,

apps/server/src/orchestration-v2/CheckpointRestoreSafety.test.ts

@@ -16,6 +16,7 @@ import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
 import * as Path from "effect/Path";
 import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
+import { GitWorkflowService } from "../git/GitWorkflowService.ts";
 import {
   CheckpointRollbackServiceV2,
   layer as rollbackLayer,
@@ -106,6 +107,7 @@ it.effect.each([
         Layer.mergeAll(
           NodeServices.layer,
           idAllocatorLayer,
+          Layer.mock(GitWorkflowService)({}),
           Layer.mock(ProjectStore.ProjectStoreV2)({
             get: () => Effect.succeed(Option.some({ workspaceRoot: parent } as never)),
           }),

apps/server/src/orchestration-v2/CheckpointRollbackService.ts

@@ -10,17 +10,20 @@ import * as DateTime from "effect/DateTime";
 import * as Effect from "effect/Effect";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
 import * as Path from "effect/Path";
 import * as Schema from "effect/Schema";
 
+import * as GitWorkflowService from "../git/GitWorkflowService.ts";
+import { resolveWorkspacePath, withWorkspaceLease } from "../workspace/workspaceLease.ts";
 import {
   isCheckpointRestoreIsolated,
   SHARED_WORKSPACE_RESTORE_MESSAGE,
 } from "./CheckpointRestoreSafety.ts";
 import { CheckpointServiceV2 } from "./CheckpointService.ts";
 import { EventSinkV2 } from "./EventSink.ts";
 import { IdAllocatorV2 } from "./IdAllocator.ts";
-import { ProjectionStoreV2 } from "./ProjectionStore.ts";
+import * as ProjectionStore from "./ProjectionStore.ts";
 import * as ProjectStore from "./ProjectStore.ts";
 import type { ProviderAdapterV2RollbackTarget } from "./ProviderAdapter.ts";
 import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
@@ -82,9 +85,10 @@ export const layer: Layer.Layer<
   CheckpointRollbackServiceV2,
   never,
   | CheckpointServiceV2
+  | GitWorkflowService.GitWorkflowService
   | EventSinkV2
   | IdAllocatorV2
-  | ProjectionStoreV2
+  | ProjectionStore.ProjectionStoreV2
   | ProviderSessionManagerV2
   | RuntimePolicyV2
   | FileSystem.FileSystem
@@ -94,23 +98,18 @@ export const layer: Layer.Layer<
   CheckpointRollbackServiceV2,
   Effect.gen(function* () {
     const checkpoints = yield* CheckpointServiceV2;
+    const gitWorkflow = yield* GitWorkflowService.GitWorkflowService;
     const eventSink = yield* EventSinkV2;
     const ids = yield* IdAllocatorV2;
-    const projections = yield* ProjectionStoreV2;
+    const projections = yield* ProjectionStore.ProjectionStoreV2;
     const sessions = yield* ProviderSessionManagerV2;
     const runtimePolicy = yield* RuntimePolicyV2;
     const fileSystem = yield* FileSystem.FileSystem;
     const projects = yield* ProjectStore.ProjectStoreV2;
     const path = yield* Path.Path;
 
-    const execute = Effect.fn("orchestrationV2.checkpointRollback.execute")(function* (input: {
-      readonly threadId: ThreadId;
-      readonly providerThreadId: ProviderThreadId;
-      readonly checkpointId: CheckpointId;
-      readonly scopeId: CheckpointScopeId;
-      readonly restoreFiles?: boolean;
-    }) {
-      const projection = yield* projections.getThreadRecords(input.threadId, [
+    const readProjection = (threadId: ThreadId) =>
+      projections.getThreadRecords(threadId, [
         "providerThreads",
         "providerSessions",
         "checkpoints",
@@ -120,6 +119,17 @@ export const layer: Layer.Layer<
         "nodes",
         "providerTurns",
       ]);
+
+    const execute = Effect.fn("orchestrationV2.checkpointRollback.execute")(function* (
+      input: {
+        readonly threadId: ThreadId;
+        readonly providerThreadId: ProviderThreadId;
+        readonly checkpointId: CheckpointId;
+        readonly scopeId: CheckpointScopeId;
+        readonly restoreFiles?: boolean;
+      },
+      projection: Effect.Success<ReturnType<typeof readProjection>>,
+    ) {
       const providerThread = projection.providerThreads.find(
         (candidate) => candidate.id === input.providerThreadId,
       );
@@ -154,6 +164,53 @@ export const layer: Layer.Layer<
         });
       }
 
+      const targetOrdinal = checkpoint.appRunOrdinal ?? 0;
+      const rollbackTarget: ProviderAdapterV2RollbackTarget =
+        targetOrdinal === 0
+          ? {
+              type: "thread_start",
+              checkpointId: checkpoint.id,
+              appRunOrdinal: 0,
+            }
+          : yield* Effect.gen(function* () {
+              const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal);
+              const targetAttempt = projection.attempts.find(
+                (attempt) => attempt.id === targetRun?.activeAttemptId,
+              );
+              const targetTurn = projection.providerTurns.find(
+                (turn) =>
+                  turn.id === targetAttempt?.providerTurnId ||
+                  turn.runAttemptId === targetAttempt?.id,
+              );
+              if (targetTurn === undefined || targetTurn.providerThreadId !== providerThread.id) {
+                return yield* new CheckpointRollbackExecutionError({
+                  reason: "provider-turn-unavailable",
+                  threadId: input.threadId,
+                  providerThreadId: input.providerThreadId,
+                  checkpointId: input.checkpointId,
+                });
+              }
+              return {
+                type: "provider_turn" as const,
+                checkpointId: checkpoint.id,
+                appRunOrdinal: targetOrdinal,
+                providerTurn: targetTurn,
+              };
+            });
+
+      const { worktreePath, branch } = projection.thread;
+      if (worktreePath != null && branch != null && !(yield* fileSystem.exists(worktreePath))) {
+        const project = yield* projects.get(projection.thread.projectId);
+        if (Option.isSome(project)) {
+          yield* gitWorkflow.pruneWorktrees({ cwd: project.value.workspaceRoot });
+          yield* gitWorkflow.createWorktree({
+            cwd: project.value.workspaceRoot,
+            refName: branch,
+            path: worktreePath,
+          });
+        }
+      }
+
       if (
         input.restoreFiles !== false &&
         !(yield* isCheckpointRestoreIsolated(projection.thread, scope, {
@@ -195,7 +252,6 @@ export const layer: Layer.Layer<
             }),
       });
 
-      const targetOrdinal = checkpoint.appRunOrdinal ?? 0;
       // Stopped and failed runs after the target leave the provider
       // conversation too, so they must not stay visible.
       const runsToRollback = projection.runs.filter(
@@ -221,38 +277,6 @@ export const layer: Layer.Layer<
           turn.providerThreadId === providerThread.id &&
           (turn.runAttemptId === null || !rolledBackAttemptIds.has(turn.runAttemptId)),
       );
-      const rollbackTarget: ProviderAdapterV2RollbackTarget =
-        targetOrdinal === 0
-          ? {
-              type: "thread_start",
-              checkpointId: checkpoint.id,
-              appRunOrdinal: 0,
-            }
-          : yield* Effect.gen(function* () {
-              const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal);
-              const targetAttempt = projection.attempts.find(
-                (attempt) => attempt.id === targetRun?.activeAttemptId,
-              );
-              const targetTurn = projection.providerTurns.find(
-                (turn) =>
-                  turn.id === targetAttempt?.providerTurnId ||
-                  turn.runAttemptId === targetAttempt?.id,
-              );
-              if (targetTurn === undefined || targetTurn.providerThreadId !== providerThread.id) {
-                return yield* new CheckpointRollbackExecutionError({
-                  reason: "provider-turn-unavailable",
-                  threadId: input.threadId,
-                  providerThreadId: input.providerThreadId,
-                  checkpointId: input.checkpointId,
-                });
-              }
-              return {
-                type: "provider_turn" as const,
-                checkpointId: checkpoint.id,
-                appRunOrdinal: targetOrdinal,
-                providerTurn: targetTurn,
-              };
-            });
 
       const snapshot =
         runsToRollback.length === 0
@@ -344,7 +368,38 @@ export const layer: Layer.Layer<
 
     return CheckpointRollbackServiceV2.of({
       execute: (input) =>
-        execute(input).pipe(
+        Effect.gen(function* () {
+          const resolvePath = (worktreePath: string | null) =>
+            worktreePath == null
+              ? Effect.succeed(null)
+              : resolveWorkspacePath(worktreePath).pipe(
+                  Effect.provideService(FileSystem.FileSystem, fileSystem),
+                  Effect.provideService(Path.Path, path),
+                );
+          const initial = yield* projections.getThreadRecords(input.threadId, []);
+          let worktreePath = yield* resolvePath(initial.thread.worktreePath);
+          while (true) {
+            const effect = Effect.gen(function* () {
+              const storedProjection = yield* readProjection(input.threadId);
+              const freshPath = yield* resolvePath(storedProjection.thread.worktreePath);
+              if (freshPath !== worktreePath) return freshPath;
+              const projection =
+                freshPath === null
+                  ? storedProjection
+                  : {
+                      ...storedProjection,
+                      thread: { ...storedProjection.thread, worktreePath: freshPath },
+                    };
+              yield* execute(input, projection);
+              return undefined;
+            });
+            const freshPath = yield* worktreePath === null
+              ? effect
+              : withWorkspaceLease(worktreePath, effect);
+            if (freshPath === undefined) return;
+            worktreePath = freshPath;
+          }
+        }).pipe(
           Effect.mapError((cause) =>
             isCheckpointRollbackExecutionError(cause)
               ? cause