diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index 2097bd4ef79a..d2083b1545f5 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -23,6 +23,7 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme"; import { useThemeColor } from "../../lib/useThemeColor"; import type { PendingNewTask } from "../../state/use-pending-new-tasks"; import { terminalEnvironment } from "../../state/terminal"; +import { threadEnvironment } from "../../state/threads"; import { useAtomCommand } from "../../state/use-atom-command"; import { useThreadPr, type ThreadPr } from "../../state/use-thread-pr"; import type { HomeGroupDisplayAction } from "../home/homeListItems"; @@ -418,6 +419,11 @@ const THREAD_ROW_MENU_ACTIONS: MenuAction[] = [ { id: "delete", title: "Delete", image: "trash", attributes: { destructive: true } }, ]; +const FAILED_CLEANUP_MENU_ACTIONS: MenuAction[] = [ + { id: "retry-worktree-cleanup", title: "Retry", image: "arrow.clockwise" }, + { id: "keep-worktree", title: "Keep worktree", image: "externaldrive" }, +]; + export const ThreadListRow = memo(function ThreadListRow(props: { readonly variant: ThreadListVariant; readonly thread: EnvironmentThreadShell; @@ -459,8 +465,16 @@ export const ThreadListRow = memo(function ThreadListRow(props: { const { thread, onSelectThread, onArchiveThread, onDeleteThread, onRegenerateThreadTitle } = props; + const cleanupFailed = thread.worktreeCleanup?.status === "failed"; + const cleanupPending = thread.worktreeCleanup != null && !cleanupFailed; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); + const retryWorktreeCleanup = useAtomCommand(threadEnvironment.retryWorktreeCleanup, { + reportFailure: false, + }); + const abandonWorktreeCleanup = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { + reportFailure: false, + }); const status = resolveThreadStatus(thread); const pr = useThreadPr(thread, props.projectCwd); const timestamp = relativeTime( @@ -504,6 +518,48 @@ export const ThreadListRow = memo(function ThreadListRow(props: { ); } }, [closeTerminal, runningAction, thread.environmentId, thread.id]); + const handleRetryWorktreeCleanup = useCallback(async () => { + const result = await retryWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }); + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not retry worktree cleanup", + error instanceof Error ? error.message : "The worktree cleanup could not be retried.", + ); + } + }, [retryWorktreeCleanup, thread.environmentId, thread.id]); + const handleKeepWorktree = useCallback(() => { + Alert.alert( + "Keep worktree?", + "LastCode will stop trying to remove this worktree. You can remove it manually later.", + [ + { text: "Cancel", style: "cancel" }, + { + text: "Keep worktree", + style: "destructive", + onPress: () => { + void abandonWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }).then((result) => { + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not keep worktree", + error instanceof Error + ? error.message + : "The worktree cleanup could not be dismissed.", + ); + } + }); + }, + }, + ], + ); + }, [abandonWorktreeCleanup, thread.environmentId, thread.id]); const menuActions = useMemo( () => [ THREAD_ROW_MENU_ACTIONS[0]!, @@ -539,9 +595,18 @@ export const ThreadListRow = memo(function ThreadListRow(props: { if (nativeEvent.event === "archive") handleArchive(); if (nativeEvent.event === "regenerate-title") handleRegenerateTitle(); if (nativeEvent.event === "cancel-action") void handleCancelAction(); + if (nativeEvent.event === "retry-worktree-cleanup") void handleRetryWorktreeCleanup(); + if (nativeEvent.event === "keep-worktree") handleKeepWorktree(); if (nativeEvent.event === "delete") handleDelete(); }, - [handleArchive, handleCancelAction, handleDelete, handleRegenerateTitle], + [ + handleArchive, + handleCancelAction, + handleDelete, + handleKeepWorktree, + handleRegenerateTitle, + handleRetryWorktreeCleanup, + ], ); const statusPill = effectiveStatus ? ( @@ -595,11 +660,18 @@ export const ThreadListRow = memo(function ThreadListRow(props: { const rowContent = (close: () => void) => compact ? ( { + if (cleanupFailed) return; close(); onSelectThread(thread); }} @@ -648,13 +720,19 @@ export const ThreadListRow = memo(function ThreadListRow(props: { ) : ( setHovered(true)} onHoverOut={() => setHovered(false)} onPress={() => { + if (cleanupFailed) return; close(); onSelectThread(thread); }} @@ -711,6 +789,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { return ( {rowContent(close)} diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 664c8d9287e7..55557c2a72ea 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -24,16 +24,19 @@ import { relativeTime } from "../../lib/time"; import { useThemeColor } from "../../lib/useThemeColor"; import type { PendingNewTask } from "../../state/use-pending-new-tasks"; import { terminalEnvironment } from "../../state/terminal"; +import { threadEnvironment } from "../../state/threads"; import { useAtomCommand } from "../../state/use-atom-command"; import { useThreadPr } from "../../state/use-thread-pr"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; import { buildThreadTitleRegenerationMenuItems } from "./thread-title-regeneration-menu"; +import { resolveWorktreeCleanupStatus } from "./threadPresentation"; import { resolveThreadListV2SnoozeMenuSelection, resolveThreadListV2SnoozeGateExpiryMs, resolveThreadListV2Status, resolveThreadListV2SwipeActions, + resolveThreadListV2CleanupActions, type ThreadListV2Status, } from "./threadListV2"; import { ThreadSearchMatchExcerpt } from "./thread-search-match"; @@ -91,6 +94,11 @@ const LEGACY_MENU_ACTIONS: MenuAction[] = [ { id: "delete", title: "Delete", image: "trash", attributes: { destructive: true } }, ]; +const FAILED_CLEANUP_MENU_ACTIONS: MenuAction[] = [ + { id: "retry-worktree-cleanup", title: "Retry", image: "arrow.clockwise" }, + { id: "keep-worktree", title: "Keep worktree", image: "externaldrive" }, +]; + /** Rounded-row radius shared with the v1 sidebar rows. */ const SIDEBAR_V2_ROW_RADIUS = 12; @@ -406,7 +414,15 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const snoozedRow = props.snoozed === true; const pinnedRow = props.pinned === true; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; + const cleanupFailed = resolveThreadListV2CleanupActions(thread.worktreeCleanup).length > 0; + const cleanupPending = thread.worktreeCleanup != null && !cleanupFailed; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); + const retryWorktreeCleanup = useAtomCommand(threadEnvironment.retryWorktreeCleanup, { + reportFailure: false, + }); + const abandonWorktreeCleanup = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { + reportFailure: false, + }); const pr = useThreadPr(thread, props.projectCwd ?? props.project?.workspaceRoot ?? null); const prState = pr?.state ?? null; @@ -428,7 +444,10 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const selected = props.selected === true; const status = resolveThreadListV2Status(thread); - const statusLabel = STATUS_LABEL_BY_STATUS[status]; + const cleanupStatus = resolveWorktreeCleanupStatus(thread); + const statusLabel = cleanupStatus + ? { label: cleanupStatus.label, className: cleanupStatus.textClassName } + : STATUS_LABEL_BY_STATUS[status]; const timeLabel = threadTimeLabel(thread); const handleDelete = useCallback(() => onDeleteThread(thread), [onDeleteThread, thread]); @@ -468,6 +487,48 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { ); } }, [closeTerminal, runningAction, thread.environmentId, thread.id]); + const handleRetryWorktreeCleanup = useCallback(async () => { + const result = await retryWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }); + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not retry worktree cleanup", + error instanceof Error ? error.message : "The worktree cleanup could not be retried.", + ); + } + }, [retryWorktreeCleanup, thread.environmentId, thread.id]); + const handleKeepWorktree = useCallback(() => { + Alert.alert( + "Keep worktree?", + "LastCode will stop trying to remove this worktree. You can remove it manually later.", + [ + { text: "Cancel", style: "cancel" }, + { + text: "Keep worktree", + style: "destructive", + onPress: () => { + void abandonWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }).then((result) => { + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not keep worktree", + error instanceof Error + ? error.message + : "The worktree cleanup could not be dismissed.", + ); + } + }); + }, + }, + ], + ); + }, [abandonWorktreeCleanup, thread.environmentId, thread.id]); // Swipe: the v2 primary action is the lifecycle transition. Every settled // row can un-settle — explicit settles clear the override, auto-settled @@ -628,6 +689,8 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { if (nativeEvent.event === "archive") handleArchive(); if (nativeEvent.event === "regenerate-title") handleRegenerateTitle(); if (nativeEvent.event === "cancel-action") void handleCancelAction(); + if (nativeEvent.event === "retry-worktree-cleanup") void handleRetryWorktreeCleanup(); + if (nativeEvent.event === "keep-worktree") handleKeepWorktree(); if (nativeEvent.event === "delete") handleDelete(); const snoozeSelection = resolveThreadListV2SnoozeMenuSelection({ event: nativeEvent.event, @@ -644,7 +707,9 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { handleArchive, handleCancelAction, handleDelete, + handleKeepWorktree, handleRegenerateTitle, + handleRetryWorktreeCleanup, handleMovePinnedDown, handleMovePinnedUp, handlePin, @@ -848,10 +913,11 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { accessibilityHint={swipeAccessibilityHint} accessibilityLabel={thread.title} accessibilityRole="button" - accessibilityState={{ selected }} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} onPress={() => { close(); - onSelectThread(thread); + if (!cleanupFailed) onSelectThread(thread); }} style={ sidebarPane @@ -888,11 +954,12 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { accessibilityHint={swipeAccessibilityHint} accessibilityLabel={thread.title} accessibilityRole="button" - accessibilityState={{ selected }} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} className={sidebarPane ? undefined : "bg-screen"} onPress={() => { close(); - onSelectThread(thread); + if (!cleanupFailed) onSelectThread(thread); }} style={ sidebarPane @@ -971,6 +1038,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { sidebarPane ? { borderRadius: SIDEBAR_V2_ROW_RADIUS, overflow: "hidden" } : undefined } enableTrackpadSwipe + enabled={!cleanupPending && !cleanupFailed} // Full swipe commits the advertised lifecycle action (Settle / // Un-settle), never the secondary snooze action. fullSwipeAction="primary" @@ -987,18 +1055,22 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { {(close) => ( {rowContent(close)} diff --git a/apps/mobile/src/features/threads/threadListV2.test.ts b/apps/mobile/src/features/threads/threadListV2.test.ts index 9f3135c50672..c272c9414bf1 100644 --- a/apps/mobile/src/features/threads/threadListV2.test.ts +++ b/apps/mobile/src/features/threads/threadListV2.test.ts @@ -17,13 +17,14 @@ import { buildThreadListV2Items, buildThreadListV2ListItems, resolveThreadListV2Enabled, + resolveThreadListV2CleanupActions, resolveThreadListV2SnoozeMenuSelection, resolveThreadListV2SnoozeGateExpiryMs, resolveThreadListV2Status, resolveThreadListV2SwipeActions, sortThreadsForListV2, } from "./threadListV2"; -import { resolveThreadStatus } from "./threadPresentation"; +import { resolveThreadStatus, resolveWorktreeCleanupStatus } from "./threadPresentation"; const environmentId = EnvironmentId.make("environment-1"); @@ -127,6 +128,53 @@ describe("resolveThreadListV2Enabled", () => { }); describe("resolveThreadListV2Status", () => { + it("shows durable cleanup before agent status", () => { + const deleting = makeThread({ + id: ThreadId.make("cleanup"), + title: "Cleanup", + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + }, + }); + expect(resolveThreadStatus(deleting)).toMatchObject({ + kind: "cleanup-deleting", + label: "Deleting", + pulse: false, + }); + expect(resolveWorktreeCleanupStatus(deleting)).toMatchObject({ + kind: "cleanup-deleting", + label: "Deleting", + }); + expect( + resolveThreadStatus({ + ...deleting, + worktreeCleanup: { + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }, + }), + ).toMatchObject({ kind: "cleanup-failed", label: "Cleanup failed" }); + expect( + resolveWorktreeCleanupStatus({ + ...deleting, + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("blocking"), + }, + }), + ).toMatchObject({ kind: "cleanup-queued", label: "Deleting (Queued)" }); + }); + it("prioritizes approval over a running session", () => { const thread = makeThread({ id: ThreadId.make("t"), @@ -169,6 +217,41 @@ describe("resolveThreadListV2Status", () => { }); }); +describe("resolveThreadListV2CleanupActions", () => { + it("keeps failed cleanup tombstones recoverable from the mobile menu", () => { + expect( + resolveThreadListV2CleanupActions({ + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }), + ).toEqual(["retry-worktree-cleanup", "keep-worktree"]); + }); + + it("keeps queued and active cleanup tombstones menu-inert", () => { + expect( + resolveThreadListV2CleanupActions({ + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("blocking"), + }), + ).toEqual([]); + expect( + resolveThreadListV2CleanupActions({ + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + }), + ).toEqual([]); + }); +}); + describe("resolveThreadListV2SwipeActions", () => { it("offers settle and snooze for an active snoozable thread", () => { expect( @@ -280,6 +363,75 @@ describe("sortThreadsForListV2", () => { }); describe("buildThreadListV2Items", () => { + it("keeps all cleanup tombstones in the visible active block", () => { + const cleanupTombstones = [ + makeThread({ + id: ThreadId.make("cleanup-queued"), + title: "Queued cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/queued", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("cleanup-blocker"), + }, + }), + makeThread({ + id: ThreadId.make("cleanup-deleting"), + title: "Deleting cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/deleting", + startedAt: NOW, + }, + }), + makeThread({ + id: ThreadId.make("cleanup-failed"), + title: "Failed cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/failed", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }, + }), + ]; + const layout = buildThreadListV2Items({ + threads: cleanupTombstones, + environmentId: null, + searchQuery: "", + now: NOW, + }); + + expect(layout.items.map((item) => item.thread.id)).toEqual([ + "cleanup-deleting", + "cleanup-failed", + "cleanup-queued", + ]); + expect(layout.items.map((item) => item.variant)).toEqual(["card", "card", "card"]); + expect(layout.items.map((item) => item.pinned)).toEqual([false, false, false]); + expect(layout.snoozedCount).toBe(0); + expect(layout.settledCount).toBe(0); + }); + it("keeps a merged thread active when auto-settle on merge is off", () => { const merged = makeThread({ id: ThreadId.make("merged"), title: "Merged" }); const layout = buildThreadListV2Items({ diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index d0573d997242..261d97b9a06d 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -30,6 +30,15 @@ export { snoozeWakeLabel }; export type ThreadListV2Status = "approval" | "input" | "working" | "waiting" | "failed" | "ready"; export type ThreadListV2SwipeAction = "archive" | "settle" | "unsettle" | "snooze" | "unsnooze"; +export type ThreadListV2CleanupAction = "retry-worktree-cleanup" | "keep-worktree"; + +/** Failed cleanup tombstones stay reachable on mobile through recovery actions. */ +export function resolveThreadListV2CleanupActions( + cleanup: EnvironmentThreadShell["worktreeCleanup"], +): readonly ThreadListV2CleanupAction[] { + return cleanup?.status === "failed" ? ["retry-worktree-cleanup", "keep-worktree"] : []; +} + export function resolveThreadListV2SnoozeMenuSelection(input: { readonly event: string; readonly displayedPresets: ReadonlyArray; @@ -392,6 +401,13 @@ export function buildThreadListV2Items(input: { const supportsSnooze = input.snoozeEnvironmentIds?.has(thread.environmentId) ?? true; const changeRequest = input.changeRequestByKey?.get(`${thread.environmentId}:${thread.id}`) ?? null; + // Cleanup tombstones are deleted-thread recovery state, not lifecycle + // state. Keep them in the immediately visible active block regardless of + // stale snooze, settle, or pin metadata retained on the thread. + if (thread.worktreeCleanup != null) { + active.push(thread); + continue; + } // Snooze outranks settlement and pinning until the thread wakes. if (supportsSnooze && effectiveSnoozed(thread, { now: snoozeNow })) { snoozed.push(thread); diff --git a/apps/mobile/src/features/threads/threadPresentation.ts b/apps/mobile/src/features/threads/threadPresentation.ts index c556dc65e3ac..0bd0cfa51371 100644 --- a/apps/mobile/src/features/threads/threadPresentation.ts +++ b/apps/mobile/src/features/threads/threadPresentation.ts @@ -14,6 +14,9 @@ export type ThreadStatusKind = | "waiting" | "connecting" | "error" + | "cleanup-deleting" + | "cleanup-queued" + | "cleanup-failed" | "plan-ready"; export interface ThreadStatusPresentation extends StatusTone { @@ -50,6 +53,42 @@ function isLatestTurnSettled( export function resolveThreadStatus( thread: EnvironmentThreadShell, ): ThreadStatusPresentation | null { + if (thread.worktreeCleanup?.status === "failed") { + return { + kind: "cleanup-failed", + label: "Cleanup failed", + pillClassName: "bg-rose-500/12 dark:bg-rose-500/16", + textClassName: "text-rose-700 dark:text-rose-300", + iconColor: "#ff453a", + iconBackground: "rgba(255,69,58,0.22)", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "queued") { + return { + kind: "cleanup-queued", + label: "Deleting (Queued)", + pillClassName: "bg-orange-500/12 dark:bg-orange-500/16", + textClassName: "text-orange-700 dark:text-orange-300", + iconColor: "#ff9f0a", + iconBackground: "rgba(255,159,10,0.22)", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "deleting") { + return { + kind: "cleanup-deleting", + label: "Deleting", + pillClassName: "bg-orange-500/12 dark:bg-orange-500/16", + textClassName: "text-orange-700 dark:text-orange-300", + iconColor: "#ff9f0a", + iconBackground: "rgba(255,159,10,0.22)", + pulse: false, + }; + } + if (thread.hasPendingApprovals) { return { kind: "pending-approval", @@ -140,3 +179,20 @@ export function resolveThreadStatus( return null; } + +/** + * Returns the durable cleanup status when a thread is being deleted. Mobile + * list variants use this shared presentation so cleanup state cannot fall + * through to the ordinary agent-status labels. + */ +export function resolveWorktreeCleanupStatus( + thread: EnvironmentThreadShell, +): ThreadStatusPresentation | null { + if (thread.worktreeCleanup == null) return null; + const status = resolveThreadStatus(thread); + return status?.kind === "cleanup-failed" || + status?.kind === "cleanup-queued" || + status?.kind === "cleanup-deleting" + ? status + : null; +} diff --git a/apps/server/src/environment/ServerEnvironment.test.ts b/apps/server/src/environment/ServerEnvironment.test.ts index 0b6c2a527995..7c22df14d41f 100644 --- a/apps/server/src/environment/ServerEnvironment.test.ts +++ b/apps/server/src/environment/ServerEnvironment.test.ts @@ -93,6 +93,7 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => { expect(second.capabilities.pullRequests).toBe(true); expect(second.capabilities.threadTitleRegeneration).toBe(true); expect(second.capabilities.threadAnnotations).toBe(true); + expect(second.capabilities.threadWorktreeCleanup).toBe(true); expect(second.capabilities.agentActivityPublishing).toBe(false); }), ); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 11dd4319beb2..d14b849f5cdb 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -153,6 +153,7 @@ export const make = Effect.gen(function* () { threadPinReorder: true, threadTitleRegeneration: true, threadAnnotations: true, + threadWorktreeCleanup: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), ...(serverSelfUpdate === "boot-service" ? { serverSelfUpdateProgress: true } : {}), }, diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 5575bb40c20f..d350e85e4f66 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -635,6 +635,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti titleRegenerationRequestId: null, titleRegenerationStartedAt: null, annotation: null, + worktreeCleanup: null, latestUserMessageId: null, latestUserMessageAt: null, pendingApprovalCount: 0, @@ -874,11 +875,27 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti yield* projectionThreadRepository.upsert({ ...existingRow.value, deletedAt: event.payload.deletedAt, + worktreeCleanup: event.payload.worktreeCleanup ?? null, updatedAt: event.payload.deletedAt, }); return; } + case "thread.worktree-cleanup-updated": { + const existingRow = yield* projectionThreadRepository.getById({ + threadId: event.payload.threadId, + }); + if (Option.isNone(existingRow)) { + return; + } + yield* projectionThreadRepository.upsert({ + ...existingRow.value, + worktreeCleanup: event.payload.cleanup, + updatedAt: event.payload.updatedAt, + }); + return; + } + case "thread.message-sent": case "thread.proposed-plan-upserted": case "thread.activity-appended": diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 9cad0607b1b8..8217ed56776d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -1630,6 +1630,32 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { const shellSnapshot = yield* snapshotQuery.getShellSnapshot(); assert.equal(shellSnapshot.projects.length, 0); assert.equal(shellSnapshot.threads.length, 0); + + yield* sql` + UPDATE projection_projects + SET deleted_at = NULL + WHERE project_id = 'project-deleted' + `; + yield* sql` + UPDATE projection_threads + SET + worktree_path = '/tmp/deleted-project-worktrees/thread-deleted', + worktree_cleanup_json = '{"status":"deleting","repositoryRoot":"/tmp/deleted-project","worktreePath":"/tmp/deleted-project-worktrees/thread-deleted","startedAt":"2026-04-05T00:00:05.000Z"}' + WHERE thread_id = 'thread-deleted' + `; + + const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); + assert.equal(cleanupShellSnapshot.projects.length, 1); + assert.deepStrictEqual(cleanupShellSnapshot.threads[0]?.worktreeCleanup, { + status: "deleting", + repositoryRoot: "/tmp/deleted-project", + worktreePath: "/tmp/deleted-project-worktrees/thread-deleted", + startedAt: "2026-04-05T00:00:05.000Z", + }); + const cleanupDetail = yield* snapshotQuery.getThreadDetailById( + ThreadId.make("thread-deleted"), + ); + assert.equal(cleanupDetail._tag, "None"); }), ); @@ -2444,12 +2470,14 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) = const detailWithPinnedRequests = yield* snapshotQuery.getThreadDetailById(threadW); assert.equal(detailWithPinnedRequests._tag, "Some"); if (detailWithPinnedRequests._tag === "Some") { - const ids = detailWithPinnedRequests.value.activities.map((activity) => activity.id); + const ids = new Set( + detailWithPinnedRequests.value.activities.map((activity) => activity.id), + ); assert.equal(detailWithPinnedRequests.value.activities.length, 503); - assert.equal(ids.includes(asEventId("approval-old")), true); - assert.equal(ids.includes(asEventId("user-input-old")), true); - assert.equal(ids.includes(asEventId("user-input-closed")), false); - assert.equal(ids.includes(asEventId("user-input-tied-z-request")), true); + assert.equal(ids.has(asEventId("approval-old")), true); + assert.equal(ids.has(asEventId("user-input-old")), true); + assert.equal(ids.has(asEventId("user-input-closed")), false); + assert.equal(ids.has(asEventId("user-input-tied-z-request")), true); } const windowWithPinnedRequests = yield* snapshotQuery.getThreadDetailSnapshot(threadW, { @@ -2457,12 +2485,14 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) = }); assert.equal(windowWithPinnedRequests._tag, "Some"); if (windowWithPinnedRequests._tag === "Some") { - const ids = windowWithPinnedRequests.value.thread.activities.map((activity) => activity.id); + const ids = new Set( + windowWithPinnedRequests.value.thread.activities.map((activity) => activity.id), + ); assert.equal(windowWithPinnedRequests.value.thread.activities.length, 503); - assert.equal(ids.includes(asEventId("approval-old")), true); - assert.equal(ids.includes(asEventId("user-input-old")), true); - assert.equal(ids.includes(asEventId("user-input-closed")), false); - assert.equal(ids.includes(asEventId("user-input-tied-z-request")), true); + assert.equal(ids.has(asEventId("approval-old")), true); + assert.equal(ids.has(asEventId("user-input-old")), true); + assert.equal(ids.has(asEventId("user-input-closed")), false); + assert.equal(ids.has(asEventId("user-input-tied-z-request")), true); } }), ); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 42db0c4e2c5d..b1eafe6a1cc3 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -26,6 +26,7 @@ import { ProjectId, ThreadId, ThreadAnnotation, + ThreadWorktreeCleanup, } from "@t3tools/contracts"; import * as Arr from "effect/Array"; import * as Effect from "effect/Effect"; @@ -92,6 +93,7 @@ const ProjectionThreadDbRowSchema = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), annotation: Schema.NullOr(Schema.fromJsonString(ThreadAnnotation)), + worktreeCleanup: Schema.optional(Schema.NullOr(Schema.fromJsonString(ThreadWorktreeCleanup))), }), ); const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( @@ -444,6 +446,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -482,6 +485,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -489,8 +493,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { has_actionable_proposed_plan AS "hasActionableProposedPlan", deleted_at AS "deletedAt" FROM projection_threads - WHERE deleted_at IS NULL - AND archived_at IS NULL + WHERE (deleted_at IS NULL AND archived_at IS NULL) + OR worktree_cleanup_json IS NOT NULL ORDER BY project_id ASC, created_at ASC, thread_id ASC `, }); @@ -522,6 +526,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -966,6 +971,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -974,8 +980,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { deleted_at AS "deletedAt" FROM projection_threads WHERE thread_id = ${threadId} - AND deleted_at IS NULL - AND archived_at IS NULL + AND ((deleted_at IS NULL AND archived_at IS NULL) + OR worktree_cleanup_json IS NOT NULL) LIMIT 1 `, }); @@ -1727,6 +1733,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), deletedAt: row.deletedAt, messages: messagesByThread.get(row.threadId) ?? [], proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], @@ -1936,6 +1943,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), latestUserMessageId: row.latestUserMessageId, + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), deletedAt: row.deletedAt, messages: [], proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], @@ -2051,7 +2059,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { : Result.failVoid, ), threads: Arr.filterMap(threadRows, (row) => - row.deletedAt === null + row.deletedAt === null || row.worktreeCleanup != null ? Result.succeed({ id: row.threadId, projectId: row.projectId, @@ -2073,6 +2081,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null + ? { worktreeCleanup: row.worktreeCleanup } + : {}), session: sessionByThread.get(row.threadId) ?? null, latestUserMessageAt: row.latestUserMessageAt, hasPendingApprovals: row.pendingApprovalCount > 0, @@ -2220,6 +2231,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), session: sessionByThread.get(row.threadId) ?? null, latestUserMessageAt: row.latestUserMessageAt, hasPendingApprovals: row.pendingApprovalCount > 0, @@ -2501,6 +2513,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: threadRow.value.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(threadRow.value), ...(threadRow.value.annotation !== null ? { annotation: threadRow.value.annotation } : {}), + ...(threadRow.value.worktreeCleanup != null + ? { worktreeCleanup: threadRow.value.worktreeCleanup } + : {}), session: Option.isSome(sessionRow) ? mapSessionRow(sessionRow.value) : null, latestUserMessageAt: threadRow.value.latestUserMessageAt, hasPendingApprovals: threadRow.value.pendingApprovalCount > 0, @@ -2608,7 +2623,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ), ]); - if (Option.isNone(threadRow)) { + if (Option.isNone(threadRow) || threadRow.value.deletedAt !== null) { return Option.none(); } diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 34b1b995a3ad..3f403a5f945e 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -1,10 +1,48 @@ -import { ThreadId } from "@t3tools/contracts"; +import { + CommandId, + EventId, + GitCommandError, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationCommand, + type OrchestrationEvent, + type ThreadWorktreeCleanup, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as DateTime from "effect/DateTime"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { it as effectIt } from "@effect/vitest"; import { describe, expect, it } from "vite-plus/test"; -import { logCleanupCauseUnlessInterrupted } from "./ThreadDeletionReactor.ts"; +import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts"; +import { ProviderAdapterProcessError } from "../../provider/Errors.ts"; +import { ProjectionProjectRepository } from "../../persistence/Services/ProjectionProjects.ts"; +import { + ProjectionThreadRepository, + type ProjectionThread, +} from "../../persistence/Services/ProjectionThreads.ts"; +import { PersistenceSqlError } from "../../persistence/Errors.ts"; +import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import * as TerminalManager from "../../terminal/Manager.ts"; +import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; +import { + logCleanupCauseUnlessInterrupted, + ThreadDeletionReactorLive, +} from "./ThreadDeletionReactor.ts"; describe("logCleanupCauseUnlessInterrupted", () => { const threadId = ThreadId.make("thread-deletion-reactor-test"); @@ -36,3 +74,678 @@ describe("logCleanupCauseUnlessInterrupted", () => { } }); }); + +function cleanupRow( + id: string, + cleanup: ThreadWorktreeCleanup, + deletedAt: string, +): ProjectionThread { + return { + threadId: ThreadId.make(id), + projectId: ProjectId.make("project-cleanup"), + title: id, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: id, + worktreePath: cleanup.worktreePath, + latestTurnId: null, + createdAt: deletedAt, + updatedAt: deletedAt, + archivedAt: null, + settledOverride: null, + settledAt: null, + snoozedUntil: null, + snoozedAt: null, + pinnedAt: null, + annotation: null, + worktreeCleanup: cleanup, + latestUserMessageId: null, + latestUserMessageAt: null, + pendingApprovalCount: 0, + pendingUserInputCount: 0, + hasActionableProposedPlan: 0, + deletedAt, + }; +} + +function deletedEventFor( + thread: ProjectionThread, + eventId: string, + sequence: number, +): Extract { + return { + sequence, + eventId: EventId.make(eventId), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make(`${eventId}-command`), + causationEventId: null, + correlationId: CommandId.make(`${eventId}-correlation`), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; +} + +function cleanupUpdatedEventFor( + thread: ProjectionThread, + eventId: string, + sequence: number, +): Extract { + return { + sequence, + eventId: EventId.make(eventId), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.worktree-cleanup-updated", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make(`${eventId}-command`), + causationEventId: null, + correlationId: CommandId.make(`${eventId}-correlation`), + metadata: {}, + payload: { + threadId: thread.threadId, + cleanup: thread.worktreeCleanup!, + updatedAt: "2026-08-23T00:00:00.000Z", + }, + }; +} + +describe("durable worktree cleanup", () => { + effectIt.live("tears down the thread before removing its worktree and retries completion", () => + Effect.gen(function* () { + const thread = cleanupRow( + "cleanup-event", + { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/worktrees/event", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const deletedEvent: Extract = { + sequence: 1, + eventId: EventId.make("event-thread-deleted"), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make("command-thread-deleted"), + causationEventId: null, + correlationId: CommandId.make("command-thread-deleted"), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; + const rows = new Map([[thread.threadId, thread]]); + const operations: string[] = []; + const teardownStarted = yield* Deferred.make(); + const releaseTeardown = yield* Deferred.make(); + const removed = yield* Deferred.make(); + const completionDispatchFailed = yield* Deferred.make(); + let completionDispatchAttempts = 0; + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.make(deletedEvent), + latestSequence: Effect.succeed(1), + readEvents: () => Stream.empty, + dispatch: (command) => { + if ( + command.type === "thread.worktree-cleanup.update" && + command.cleanup === null && + completionDispatchAttempts++ === 0 + ) { + return Deferred.succeed(completionDispatchFailed, undefined).pipe( + Effect.andThen( + Effect.fail( + new PersistenceSqlError({ + operation: "test.dispatchCleanup", + detail: "transient persistence failure", + }), + ), + ), + ); + } + if (command.type === "thread.worktree-cleanup.update") { + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: 2 }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path }) => + Effect.sync(() => operations.push(`remove:${path}`)).pipe( + Effect.andThen(Deferred.succeed(removed, undefined)), + ), + }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => void operations.push(`stop:${threadId}`)).pipe( + Effect.andThen(Deferred.succeed(teardownStarted, undefined)), + Effect.andThen(Deferred.await(releaseTeardown)), + ), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => void operations.push(`close:${threadId}`)), + }), + NodeServices.layer, + ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(testDependencies), + Layer.merge(testDependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Deferred.await(teardownStarted); + const drainCompleted = yield* Deferred.make(); + const drain = yield* Effect.forkChild( + reactor.drain.pipe(Effect.andThen(Deferred.succeed(drainCompleted, undefined))), + ); + expect(yield* Deferred.isDone(drainCompleted)).toBe(false); + yield* Deferred.succeed(releaseTeardown, undefined); + yield* Deferred.await(removed); + yield* Deferred.await(completionDispatchFailed); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(drain); + }).pipe(Effect.provide(testLayer)); + + expect(operations).toEqual([ + `stop:${thread.threadId}`, + `close:${thread.threadId}`, + "remove:/worktrees/event", + ]); + }), + ); + + effectIt.live("blocks worktree removal when teardown fails and retries failed persistence", () => + Effect.gen(function* () { + const thread = cleanupRow( + "cleanup-teardown-failed", + { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/worktrees/teardown-failed", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const deletedEvent: Extract = { + sequence: 1, + eventId: EventId.make("event-thread-deleted-teardown-failed"), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make("command-thread-deleted-teardown-failed"), + causationEventId: null, + correlationId: CommandId.make("command-thread-deleted-teardown-failed"), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; + const rows = new Map([[thread.threadId, thread]]); + const operations: string[] = []; + const teardownFailed = yield* Deferred.make(); + const failureDispatchFailed = yield* Deferred.make(); + let failureDispatchAttempts = 0; + const updates: Array< + Extract + > = []; + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.make(deletedEvent), + latestSequence: Effect.succeed(1), + readEvents: () => Stream.empty, + dispatch: (command) => { + if ( + command.type === "thread.worktree-cleanup.update" && + command.cleanup?.status === "failed" && + failureDispatchAttempts++ === 0 + ) { + return Deferred.succeed(failureDispatchFailed, undefined).pipe( + Effect.andThen( + Effect.fail( + new PersistenceSqlError({ + operation: "test.dispatchCleanupFailure", + detail: "transient persistence failure", + }), + ), + ), + ); + } + if (command.type === "thread.worktree-cleanup.update") { + updates.push(command); + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: updates.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: () => Effect.sync(() => operations.push("remove-worktree")), + }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => operations.push(`stop:${threadId}`)).pipe( + Effect.andThen(Deferred.succeed(teardownFailed, undefined)), + Effect.andThen( + Effect.fail( + new ProviderAdapterProcessError({ + provider: "codex", + threadId: String(threadId), + detail: "provider process did not stop", + }), + ), + ), + ), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => operations.push(`close:${threadId}`)), + }), + NodeServices.layer, + ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(testDependencies), + Layer.merge(testDependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + const drain = yield* Effect.forkChild(reactor.drain); + yield* Deferred.await(teardownFailed); + yield* Deferred.await(failureDispatchFailed); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(drain); + }).pipe(Effect.provide(testLayer)); + + expect(operations).toEqual([`stop:${thread.threadId}`]); + expect(failureDispatchAttempts).toBe(2); + expect(updates).toHaveLength(1); + expect(updates[0]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("ProviderAdapterProcessError"), + }); + }), + ); + + effectIt.live("retires idle workers and serializes jobs on a recreated worker", () => + Effect.gen(function* () { + const root = "/repo"; + const first = cleanupRow( + "cleanup-retire-first", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-first", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const second = cleanupRow( + "cleanup-retire-second", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-second", + startedAt: "2026-08-23T00:00:01.000Z", + }, + "2026-08-23T00:00:01.000Z", + ); + const third = cleanupRow( + "cleanup-retire-third", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-third", + startedAt: "2026-08-23T00:00:02.000Z", + }, + "2026-08-23T00:00:02.000Z", + ); + const rows = new Map([ + [first.threadId, first], + [second.threadId, second], + [third.threadId, third], + ]); + const events = yield* PubSub.unbounded(); + const firstRemoved = yield* Deferred.make(); + const secondStarted = yield* Deferred.make(); + const thirdStarted = yield* Deferred.make(); + const releaseSecond = yield* Deferred.make(); + const removalOrder: string[] = []; + let activeRemovals = 0; + let maxActiveRemovals = 0; + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.fromPubSub(events), + latestSequence: Effect.succeed(0), + readEvents: () => Stream.empty, + dispatch: (command) => { + if (command.type === "thread.worktree-cleanup.update") { + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: removalOrder.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path }) => + Effect.gen(function* () { + activeRemovals += 1; + maxActiveRemovals = Math.max(maxActiveRemovals, activeRemovals); + if (path === first.worktreePath) { + yield* Deferred.succeed(firstRemoved, undefined); + } else if (path === second.worktreePath) { + yield* Deferred.succeed(secondStarted, undefined); + yield* Deferred.await(releaseSecond); + } else if (path === third.worktreePath) { + yield* Deferred.succeed(thirdStarted, undefined); + } + removalOrder.push(path); + }).pipe(Effect.ensuring(Effect.sync(() => (activeRemovals -= 1)))), + }), + Layer.mock(ProviderService)({ + stopSession: () => Effect.void, + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: () => Effect.void, + }), + NodeServices.layer, + ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(testDependencies), + Layer.merge(testDependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Effect.yieldNow; + yield* PubSub.publish(events, deletedEventFor(first, "event-retire-first", 1)); + yield* Deferred.await(firstRemoved); + yield* reactor.drain; + + // The first repository worker has been idle long enough to retire. + yield* TestClock.adjust("1 minute"); + yield* PubSub.publish(events, cleanupUpdatedEventFor(second, "event-retire-second", 2)); + yield* PubSub.publish(events, cleanupUpdatedEventFor(third, "event-retire-third", 3)); + yield* Deferred.await(secondStarted); + expect(yield* Deferred.isDone(thirdStarted)).toBe(false); + yield* Deferred.succeed(releaseSecond, undefined); + yield* Deferred.await(thirdStarted); + yield* reactor.drain; + }).pipe(Effect.provide(testLayer)); + + expect(removalOrder).toEqual([first.worktreePath, second.worktreePath, third.worktreePath]); + expect(maxActiveRemovals).toBe(1); + }), + ); + + effectIt.live("resumes same-repository cleanup in order and persists failures", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const activeProjectRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-active-project-root-", + }); + const aliasParent = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-active-project-alias-", + }); + const activeProjectAlias = path.join(aliasParent, "workspace"); + yield* fileSystem.symlink(activeProjectRoot, activeProjectAlias); + const root = "/repo-a"; + const existingWorktreePath = process.cwd(); + const first = cleanupRow( + "cleanup-first", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/first", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const second = cleanupRow( + "cleanup-second", + { + status: "queued", + repositoryRoot: "/repo-b", + worktreePath: existingWorktreePath, + queuedAt: "2026-08-23T00:00:01.000Z", + blockedByThreadId: first.threadId, + }, + "2026-08-23T00:00:01.000Z", + ); + const third = cleanupRow( + "cleanup-third", + { + status: "queued", + repositoryRoot: "/repo-c", + worktreePath: "/worktrees/third", + queuedAt: "2026-08-23T00:00:02.000Z", + blockedByThreadId: second.threadId, + }, + "2026-08-23T00:00:02.000Z", + ); + const fourth = cleanupRow( + "cleanup-already-removed", + { + status: "deleting", + repositoryRoot: "/repo-d", + worktreePath: "/worktrees/already-removed", + startedAt: "2026-08-23T00:00:03.000Z", + }, + "2026-08-23T00:00:03.000Z", + ); + const fifth = cleanupRow( + "cleanup-active-project-root", + { + status: "deleting", + repositoryRoot: "/repo-e", + worktreePath: activeProjectAlias, + startedAt: "2026-08-23T00:00:04.000Z", + }, + "2026-08-23T00:00:04.000Z", + ); + const activeOwner = { + threadId: ThreadId.make("active-owner"), + worktreePath: third.worktreePath ?? "/worktrees/third", + }; + const rows = new Map([ + [first.threadId, first], + [second.threadId, second], + [third.threadId, third], + [fourth.threadId, fourth], + [fifth.threadId, fifth], + ]); + const removals: string[] = []; + const operations: string[] = []; + const updates: Array< + Extract + > = []; + + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.never, + latestSequence: Effect.succeed(0), + readEvents: () => Stream.empty, + dispatch: (command) => { + if (command.type === "thread.worktree-cleanup.update") { + updates.push(command); + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: updates.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([first, second, third, fourth, fifth]), + listActiveWorktreeOwners: () => Effect.succeed([activeOwner]), + }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => + Effect.succeed([ + { + projectId: ProjectId.make("active-project"), + title: "Active project", + workspaceRoot: activeProjectRoot, + defaultModelSelection: null, + defaultThreadEnvMode: null, + scripts: [], + createdAt: "2026-08-23T00:00:00.000Z", + updatedAt: "2026-08-23T00:00:00.000Z", + deletedAt: null, + }, + ]), + }), + Layer.mock(VcsDriverRegistry.VcsDriverRegistry)({ + resolve: () => + Effect.succeed({ + kind: "git" as const, + repository: { + kind: "git" as const, + rootPath: "/checkout", + metadataPath: "/shared-repository/.git", + freshness: { + source: "live-local" as const, + observedAt: DateTime.makeUnsafe("2026-08-23T00:00:00.000Z"), + expiresAt: Option.none(), + }, + }, + driver: null as never, + }), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path, allowMissing }) => + Effect.gen(function* () { + removals.push(path); + operations.push(`remove:${path}`); + if (path === second.worktreePath) { + return yield* new GitCommandError({ + operation: "remove worktree", + command: "git worktree remove", + cwd: root, + detail: "permission denied", + }); + } + if (path === fourth.worktreePath && allowMissing !== true) { + return yield* new GitCommandError({ + operation: "remove worktree", + command: "git worktree remove", + cwd: root, + detail: "not a working tree", + }); + } + }), + }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => void operations.push(`stop:${threadId}`)), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => void operations.push(`close:${threadId}`)), + }), + NodeServices.layer, + ); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(dependencies), + Layer.merge(dependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* reactor.drain; + }).pipe(Effect.provide(testLayer)); + + expect(removals).toEqual([ + "/worktrees/first", + second.worktreePath, + "/worktrees/already-removed", + ]); + expect(operations).toEqual([ + `stop:${first.threadId}`, + `close:${first.threadId}`, + "remove:/worktrees/first", + `stop:${second.threadId}`, + `close:${second.threadId}`, + `remove:${second.worktreePath}`, + `stop:${third.threadId}`, + `close:${third.threadId}`, + `stop:${fourth.threadId}`, + `close:${fourth.threadId}`, + "remove:/worktrees/already-removed", + `stop:${fifth.threadId}`, + `close:${fifth.threadId}`, + ]); + expect( + updates.map((command) => [command.threadId, command.cleanup?.status ?? "complete"]), + ).toEqual([ + [first.threadId, "complete"], + [second.threadId, "deleting"], + [second.threadId, "failed"], + [third.threadId, "deleting"], + [third.threadId, "failed"], + [fourth.threadId, "complete"], + [fifth.threadId, "failed"], + ]); + expect(updates[2]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("permission denied"), + }); + expect(updates[4]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("active-owner"), + }); + expect(updates[5]?.cleanup).toBeNull(); + expect(updates[6]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("active-project"), + }); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index a026f5ad81bd..ee23efa5d171 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -1,10 +1,24 @@ -import type { OrchestrationEvent } from "@t3tools/contracts"; -import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { CommandId, type OrchestrationEvent, type ThreadWorktreeCleanup } from "@t3tools/contracts"; +import { makeDrainableWorker, type DrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; +import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import { ProjectionProjectRepository } from "../../persistence/Services/ProjectionProjects.ts"; +import { ProjectionThreadRepository } from "../../persistence/Services/ProjectionThreads.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; @@ -13,8 +27,22 @@ import { type ThreadDeletionReactorShape, } from "../Services/ThreadDeletionReactor.ts"; import { forkParked } from "../../serverActivation.ts"; +import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts"; type ThreadDeletedEvent = Extract; +type PendingCleanup = Exclude; +type CleanupJob = { + readonly threadId: ThreadDeletedEvent["payload"]["threadId"]; + readonly cleanup: PendingCleanup; + readonly needsTeardown: boolean; +}; +type CleanupWorkerEntry = { + readonly repositoryKey: string; + readonly worker: DrainableWorker; + readonly generation: Ref.Ref; +}; + +const CLEANUP_WORKER_IDLE_TIMEOUT = Duration.minutes(1); export const logCleanupCauseUnlessInterrupted = ({ effect, @@ -39,8 +67,60 @@ export const logCleanupCauseUnlessInterrupted = ({ const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngineService; + const gitWorkflow = yield* GitWorkflowService; + const projectionProjects = yield* ProjectionProjectRepository; + const projectionThreads = yield* ProjectionThreadRepository; const providerService = yield* ProviderService; const terminalManager = yield* TerminalManager.TerminalManager; + const fileSystem = yield* FileSystem.FileSystem; + const vcsDriverRegistry = yield* Effect.serviceOption(VcsDriverRegistry.VcsDriverRegistry); + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const cleanupWorkersRef = yield* Ref.make>(new Map()); + const cleanupWorkersMutex = yield* Semaphore.make(1); + const enqueuedCleanupThreadIdsRef = yield* Ref.make>(new Set()); + const failedThreadTeardownIdsRef = yield* Ref.make>(new Set()); + + const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); + const canonicalPathForComparison = (value: string) => + fileSystem.realPath(value).pipe( + Effect.map(normalizeProjectPathForComparison), + Effect.orElseSucceed(() => normalizeProjectPathForComparison(value)), + ); + const serverCommandId = (tag: string) => + crypto.randomUUIDv4.pipe(Effect.map((uuid) => CommandId.make(`server:${tag}:${uuid}`))); + + const dispatchCleanup = Effect.fn("dispatchThreadWorktreeCleanup")(function* ( + threadId: CleanupJob["threadId"], + cleanup: ThreadWorktreeCleanup | null, + ) { + yield* orchestrationEngine.dispatch({ + type: "thread.worktree-cleanup.update", + commandId: yield* serverCommandId("worktree-cleanup-update"), + threadId, + cleanup, + }); + }); + + const cleanupPersistenceRetrySchedule = Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ); + const dispatchCleanupWithRetry = ( + threadId: CleanupJob["threadId"], + cleanup: ThreadWorktreeCleanup | null, + ) => + dispatchCleanup(threadId, cleanup).pipe( + Effect.retry({ schedule: cleanupPersistenceRetrySchedule }), + ); + + const clearFailedThreadTeardown = (threadId: CleanupJob["threadId"]) => + Ref.update(failedThreadTeardownIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.delete(threadId); + return next; + }); const stopProviderSession = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => logCleanupCauseUnlessInterrupted({ @@ -56,6 +136,14 @@ const make = Effect.gen(function* () { threadId, }); + const stopProviderSessionStrict = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + providerService + .stopSession({ threadId }) + .pipe(Effect.catchTag("ProviderSessionNotFoundError", () => Effect.void)); + + const closeThreadTerminalsStrict = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + terminalManager.close({ threadId, deleteHistory: true }); + const processThreadDeleted = Effect.fn("processThreadDeleted")(function* ( event: ThreadDeletedEvent, ) { @@ -64,36 +152,364 @@ const make = Effect.gen(function* () { yield* closeThreadTerminals(threadId); }); - const processThreadDeletedSafely = (event: ThreadDeletedEvent) => - processThreadDeleted(event).pipe( + const processThreadDeletedSafely = (event: ThreadDeletedEvent) => { + const cleanup = event.payload.worktreeCleanup; + const hasPendingWorktreeCleanup = cleanup != null && cleanup.status !== "failed"; + const teardown = hasPendingWorktreeCleanup + ? Effect.gen(function* () { + yield* stopProviderSessionStrict(event.payload.threadId); + yield* closeThreadTerminalsStrict(event.payload.threadId); + }) + : processThreadDeleted(event); + + return teardown.pipe( + Effect.tap(() => + hasPendingWorktreeCleanup ? clearFailedThreadTeardown(event.payload.threadId) : Effect.void, + ), Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { - return Effect.failCause(cause); + return Effect.interrupt; } - return Effect.logWarning("thread deletion reactor failed to process event", { - eventType: event.type, - threadId: event.payload.threadId, - cause: Cause.pretty(cause), + if (!hasPendingWorktreeCleanup || cleanup == null) { + return Effect.logWarning("thread deletion reactor failed to process event", { + eventType: event.type, + threadId: event.payload.threadId, + cause: Cause.pretty(cause), + }); + } + return Effect.gen(function* () { + yield* Ref.update(failedThreadTeardownIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.add(event.payload.threadId); + return next; + }); + const failedAt = yield* nowIso; + yield* dispatchCleanupWithRetry(event.payload.threadId, { + status: "failed", + repositoryRoot: cleanup.repositoryRoot, + ...(cleanup.repositoryKey === undefined + ? {} + : { repositoryKey: cleanup.repositoryKey }), + worktreePath: cleanup.worktreePath, + startedAt: cleanup.status === "deleting" ? cleanup.startedAt : failedAt, + failedAt, + error: Cause.pretty(cause), + }); + }); + }), + ); + }; + + const processCleanup = Effect.fn("processThreadWorktreeCleanup")(function* (job: CleanupJob) { + if (job.needsTeardown) { + yield* stopProviderSessionStrict(job.threadId); + yield* closeThreadTerminalsStrict(job.threadId); + yield* clearFailedThreadTeardown(job.threadId); + } + + const projected = yield* projectionThreads.getById({ threadId: job.threadId }); + if (Option.isNone(projected)) return; + const current = projected.value.worktreeCleanup; + if (current == null || current.status === "failed") return; + + const startedAt = yield* nowIso; + const deleting = { + status: "deleting" as const, + repositoryRoot: current.repositoryRoot, + ...(current.repositoryKey === undefined ? {} : { repositoryKey: current.repositoryKey }), + worktreePath: current.worktreePath, + startedAt: current.status === "deleting" ? current.startedAt : startedAt, + }; + if (current.status === "queued") { + yield* dispatchCleanup(job.threadId, deleting); + } + + const normalizedWorktreePath = yield* canonicalPathForComparison(deleting.worktreePath); + const activeProjects = yield* Effect.forEach( + yield* projectionProjects.listAll(), + (project) => + canonicalPathForComparison(project.workspaceRoot).pipe( + Effect.map((workspaceRoot) => ({ project, workspaceRoot })), + ), + { concurrency: "unbounded" }, + ); + const activeProject = activeProjects.find( + ({ project, workspaceRoot }) => + project.deletedAt === null && workspaceRoot === normalizedWorktreePath, + )?.project; + if (activeProject !== undefined) { + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: `Worktree '${deleting.worktreePath}' is now used as the workspace root of active project '${activeProject.projectId}'.`, + }); + return; + } + const activeOwners = yield* Effect.forEach( + yield* projectionThreads.listActiveWorktreeOwners(), + (owner) => + canonicalPathForComparison(owner.worktreePath).pipe( + Effect.map((worktreePath) => ({ owner, worktreePath })), + ), + { concurrency: "unbounded" }, + ); + const activeOwner = activeOwners.find( + ({ owner, worktreePath }) => + owner.threadId !== job.threadId && worktreePath === normalizedWorktreePath, + )?.owner; + if (activeOwner !== undefined) { + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: `Worktree '${deleting.worktreePath}' is now used by active thread '${activeOwner.threadId}'.`, + }); + return; + } + + const removal = yield* Effect.result( + gitWorkflow.removeWorktree({ + cwd: deleting.repositoryRoot, + path: deleting.worktreePath, + force: true, + allowMissing: true, + }), + ); + if (Result.isSuccess(removal)) { + yield* dispatchCleanupWithRetry(job.threadId, null); + return; + } + + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: removal.failure.message, + }); + }); + + const processCleanupSafely = (job: CleanupJob) => + processCleanup(job).pipe( + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); + const detail = Cause.pretty(cause); + return Effect.gen(function* () { + const failedAt = yield* nowIso; + yield* dispatchCleanupWithRetry(job.threadId, { + status: "failed", + repositoryRoot: job.cleanup.repositoryRoot, + ...(job.cleanup.repositoryKey === undefined + ? {} + : { repositoryKey: job.cleanup.repositoryKey }), + worktreePath: job.cleanup.worktreePath, + startedAt: job.cleanup.status === "deleting" ? job.cleanup.startedAt : failedAt, + failedAt, + error: detail, + }); }); }), ); - const worker = yield* makeDrainableWorker(processThreadDeletedSafely); + const removeEnqueuedCleanupThreadId = (threadId: CleanupJob["threadId"]) => + Ref.update(enqueuedCleanupThreadIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.delete(threadId); + return next; + }); + + const resolveCleanupRepositoryKey = Effect.fn("resolveCleanupRepositoryKey")(function* ( + cleanup: PendingCleanup, + ) { + const persistedKey = cleanup.repositoryKey; + + if (Option.isNone(vcsDriverRegistry)) { + return normalizeProjectPathForComparison(persistedKey ?? cleanup.repositoryRoot); + } + + const handle = yield* vcsDriverRegistry.value + .resolve({ cwd: cleanup.repositoryRoot }) + .pipe(Effect.option); + const metadataPath = Option.isNone(handle) ? null : handle.value.repository.metadataPath; + if (metadataPath === null) { + return normalizeProjectPathForComparison(persistedKey ?? cleanup.repositoryRoot); + } + const resolvedMetadataPath = path.isAbsolute(metadataPath) + ? path.normalize(metadataPath) + : path.resolve(cleanup.repositoryRoot, metadataPath); + const canonicalMetadataPath = yield* fileSystem + .realPath(resolvedMetadataPath) + .pipe(Effect.orElseSucceed(() => resolvedMetadataPath)); + return normalizeProjectPathForComparison(canonicalMetadataPath); + }); + + const getCleanupWorker = Effect.fn("getThreadWorktreeCleanupWorker")(function* ( + cleanup: PendingCleanup, + ) { + const repositoryKey = yield* resolveCleanupRepositoryKey(cleanup); + const existing = (yield* Ref.get(cleanupWorkersRef)).get(repositoryKey); + if (existing) return existing; + const created = yield* makeDrainableWorker((job: CleanupJob) => + processCleanupSafely(job).pipe(Effect.ensuring(removeEnqueuedCleanupThreadId(job.threadId))), + ); + const entry: CleanupWorkerEntry = { + repositoryKey, + worker: created, + generation: yield* Ref.make(0), + }; + yield* Ref.update(cleanupWorkersRef, (workers) => { + const next = new Map(workers); + next.set(repositoryKey, entry); + return next; + }); + yield* Effect.forkScoped( + Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CLEANUP_WORKER_IDLE_TIMEOUT); + const generation = yield* cleanupWorkersMutex.withPermit(Ref.get(entry.generation)); + // Drain outside the global mutex so a long cleanup for one + // repository cannot block unrelated repositories from enqueueing. + yield* entry.worker.drain; + const retired = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const current = (yield* Ref.get(cleanupWorkersRef)).get(repositoryKey); + if (current !== entry || (yield* Ref.get(entry.generation)) !== generation) { + return false; + } + yield* Ref.update(cleanupWorkersRef, (workers) => { + const next = new Map(workers); + if (next.get(repositoryKey) === entry) next.delete(repositoryKey); + return next; + }); + return true; + }), + ); + if (retired) { + yield* entry.worker.shutdown; + return; + } + } + }), + ); + return entry; + }); + + const enqueueCleanup = Effect.fn("enqueueThreadWorktreeCleanup")(function* (job: CleanupJob) { + const accepted = yield* Ref.modify(enqueuedCleanupThreadIdsRef, (threadIds) => { + if (threadIds.has(job.threadId)) return [false, threadIds] as const; + const next = new Set(threadIds); + next.add(job.threadId); + return [true, next] as const; + }); + if (!accepted) return; + + yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const entry = yield* getCleanupWorker(job.cleanup); + yield* Ref.update(entry.generation, (generation) => generation + 1); + yield* entry.worker.enqueue(job); + }), + ); + }); + + const enqueueCleanupFromEvent = (event: OrchestrationEvent) => { + if (event.type === "thread.deleted") { + const cleanup = event.payload.worktreeCleanup; + return cleanup == null || cleanup.status === "failed" + ? Effect.void + : enqueueCleanup({ + threadId: event.payload.threadId, + cleanup, + needsTeardown: false, + }); + } + if (event.type === "thread.worktree-cleanup-updated") { + const cleanup = event.payload.cleanup; + if (cleanup == null || cleanup.status === "failed") return Effect.void; + return enqueueCleanup({ + threadId: event.payload.threadId, + cleanup, + // Cleanup updates include retries after a persisted teardown failure. + // Repeating idempotent teardown is safer than relying on process-local + // memory, especially when the retry arrives after a server restart. + needsTeardown: true, + }); + } + return Effect.void; + }; + + const worker = yield* makeDrainableWorker((event: ThreadDeletedEvent) => + processThreadDeletedSafely(event).pipe( + Effect.andThen(Ref.get(failedThreadTeardownIdsRef)), + Effect.flatMap((failedThreadIds) => + failedThreadIds.has(event.payload.threadId) ? Effect.void : enqueueCleanupFromEvent(event), + ), + ), + ); + + const cleanupDrain: Effect.Effect = Effect.gen(function* () { + while (true) { + const snapshot = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const workers = yield* Ref.get(cleanupWorkersRef); + return yield* Effect.forEach(Array.from(workers.entries()), ([repositoryKey, entry]) => + Ref.get(entry.generation).pipe( + Effect.map((generation) => ({ repositoryKey, entry, generation })), + ), + ); + }), + ); + yield* Effect.forEach(snapshot, ({ entry }) => entry.worker.drain, { + concurrency: "unbounded", + }); + const stable = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const current = yield* Ref.get(cleanupWorkersRef); + if (current.size !== snapshot.length) return false; + const checks = yield* Effect.forEach(snapshot, ({ repositoryKey, entry, generation }) => { + if (current.get(repositoryKey) !== entry) return Effect.succeed(false); + return Ref.get(entry.generation).pipe(Effect.map((value) => value === generation)); + }); + return checks.every(Boolean); + }), + ); + if (stable) return; + } + }); const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () { yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { - if (event.type !== "thread.deleted") { - return Effect.void; + if (event.type === "thread.deleted") { + return worker.enqueue(event); } - return worker.enqueue(event); + return enqueueCleanupFromEvent(event); }), ); + + yield* projectionThreads.listPendingWorktreeCleanup().pipe( + Effect.flatMap((resumable) => + Effect.forEach(resumable, (thread) => { + const cleanup = thread.worktreeCleanup; + return cleanup == null || cleanup.status === "failed" + ? Effect.void + : enqueueCleanup({ threadId: thread.threadId, cleanup, needsTeardown: true }); + }), + ), + Effect.catchCause((cause) => + Effect.logWarning("thread worktree cleanup resume failed", { + cause: Cause.pretty(cause), + }), + ), + ); }); return { start, - drain: worker.drain, + drain: worker.drain.pipe(Effect.andThen(cleanupDrain)), } satisfies ThreadDeletionReactorShape; }); diff --git a/apps/server/src/orchestration/Normalizer.ts b/apps/server/src/orchestration/Normalizer.ts index 24c65900b296..ce879248b254 100644 --- a/apps/server/src/orchestration/Normalizer.ts +++ b/apps/server/src/orchestration/Normalizer.ts @@ -1,6 +1,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import { type ClientOrchestrationCommand, @@ -13,6 +14,8 @@ import { import { createAttachmentId, resolveAttachmentPath } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; import { parseBase64DataUrl } from "../imageMime.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; export const canonicalizeClientCommandTimestamps = ( @@ -50,8 +53,48 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const serverConfig = yield* ServerConfig; + const projectionSnapshotQuery = yield* Effect.serviceOption(ProjectionSnapshotQuery); + const vcsDriverRegistry = yield* Effect.serviceOption(VcsDriverRegistry.VcsDriverRegistry); const workspacePaths = yield* WorkspacePaths.WorkspacePaths; + const resolveGitCommonDir = (cwd: string) => + Effect.gen(function* () { + if (Option.isNone(vcsDriverRegistry)) return null; + const handle = yield* vcsDriverRegistry.value.resolve({ cwd }).pipe(Effect.option); + if (Option.isNone(handle) || handle.value.repository.metadataPath === null) { + return null; + } + const metadataPath = handle.value.repository.metadataPath; + const resolvedPath = path.isAbsolute(metadataPath) + ? path.normalize(metadataPath) + : path.resolve(cwd, metadataPath); + return yield* fileSystem + .realPath(resolvedPath) + .pipe(Effect.orElseSucceed(() => resolvedPath)); + }); + + const resolveProjectRepositoryKey = (projectId: string) => + Effect.gen(function* () { + if (Option.isNone(projectionSnapshotQuery)) return null; + const readModel = yield* projectionSnapshotQuery.value + .getCommandReadModel() + .pipe(Effect.option); + if (Option.isNone(readModel)) return null; + const project = readModel.value.projects.find((candidate) => candidate.id === projectId); + return project === undefined ? null : yield* resolveGitCommonDir(project.workspaceRoot); + }); + + const resolveThreadDeleteRepositoryKey = (threadId: string) => + Effect.gen(function* () { + if (Option.isNone(projectionSnapshotQuery)) return null; + const readModel = yield* projectionSnapshotQuery.value + .getCommandReadModel() + .pipe(Effect.option); + if (Option.isNone(readModel)) return null; + const thread = readModel.value.threads.find((candidate) => candidate.id === threadId); + return thread === undefined ? null : yield* resolveProjectRepositoryKey(thread.projectId); + }); + const normalizeProjectWorkspaceRoot = (workspaceRoot: string) => workspacePaths.normalizeWorkspaceRoot(workspaceRoot).pipe( Effect.mapError( @@ -100,6 +143,26 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => } satisfies OrchestrationCommand; } + if (canonicalCommand.type === "thread.delete" && canonicalCommand.deleteWorktree === true) { + const repositoryKey = yield* resolveThreadDeleteRepositoryKey(canonicalCommand.threadId); + const { repositoryKey: _clientRepositoryKey, ...commandWithoutRepositoryKey } = + canonicalCommand; + return { + ...commandWithoutRepositoryKey, + ...(repositoryKey === null ? {} : { repositoryKey }), + } satisfies OrchestrationCommand; + } + + if (canonicalCommand.type === "project.delete" && canonicalCommand.force === true) { + const repositoryKey = yield* resolveProjectRepositoryKey(canonicalCommand.projectId); + const { repositoryKey: _clientRepositoryKey, ...commandWithoutRepositoryKey } = + canonicalCommand; + return { + ...commandWithoutRepositoryKey, + ...(repositoryKey === null ? {} : { repositoryKey }), + } satisfies OrchestrationCommand; + } + if (canonicalCommand.type !== "thread.turn.start") { return canonicalCommand as OrchestrationCommand; } diff --git a/apps/server/src/orchestration/Schemas.ts b/apps/server/src/orchestration/Schemas.ts index 33c5d02d3274..99123c70b96a 100644 --- a/apps/server/src/orchestration/Schemas.ts +++ b/apps/server/src/orchestration/Schemas.ts @@ -9,6 +9,7 @@ import { ThreadRuntimeModeSetPayload as ContractsThreadRuntimeModeSetPayloadSchema, ThreadInteractionModeSetPayload as ContractsThreadInteractionModeSetPayloadSchema, ThreadDeletedPayload as ContractsThreadDeletedPayloadSchema, + ThreadWorktreeCleanupUpdatedPayload as ContractsThreadWorktreeCleanupUpdatedPayloadSchema, ThreadUnarchivedPayload as ContractsThreadUnarchivedPayloadSchema, ThreadUnsettledPayload as ContractsThreadUnsettledPayloadSchema, ThreadSnoozedPayload as ContractsThreadSnoozedPayloadSchema, @@ -42,6 +43,8 @@ export const ThreadMetaUpdatedPayload = ContractsThreadMetaUpdatedPayloadSchema; export const ThreadRuntimeModeSetPayload = ContractsThreadRuntimeModeSetPayloadSchema; export const ThreadInteractionModeSetPayload = ContractsThreadInteractionModeSetPayloadSchema; export const ThreadDeletedPayload = ContractsThreadDeletedPayloadSchema; +export const ThreadWorktreeCleanupUpdatedPayload = + ContractsThreadWorktreeCleanupUpdatedPayloadSchema; export const ThreadUnarchivedPayload = ContractsThreadUnarchivedPayloadSchema; export const ThreadUnsettledPayload = ContractsThreadUnsettledPayloadSchema; export const ThreadSnoozedPayload = ContractsThreadSnoozedPayloadSchema; diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index fea36b5717fe..e7b1f87b7248 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -103,6 +103,10 @@ const seedReadModel = Effect.gen(function* () { }); type PlannedEvent = Omit; +type PlannedThreadDeletedEvent = Omit< + Extract, + "sequence" +>; function normalizeDeleteEvent(event: PlannedEvent | ReadonlyArray) { const events = Array.isArray(event) ? event : [event]; @@ -137,6 +141,338 @@ function normalizeDeleteEvent(event: PlannedEvent | ReadonlyArray) } it.layer(NodeServices.layer)("decider deletion flows", (it) => { + it.effect("persists cleanup and queues later deletions from the same repository", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread, index) => ({ + ...thread, + branch: `cleanup-${index + 1}`, + worktreePath: `/tmp/project-delete-worktrees/cleanup-${index + 1}`, + })), + }; + + const first = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-1"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }); + const firstEvent = (Array.isArray(first) ? first[0] : first) as PlannedThreadDeletedEvent; + expect(firstEvent.type).toBe("thread.deleted"); + if (firstEvent.type !== "thread.deleted") return; + expect(firstEvent.payload.worktreeCleanup).toMatchObject({ + status: "deleting", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-1", + }); + + const afterFirst = yield* projectEvent(readModel, { ...firstEvent, sequence: 4 }); + const repeated = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-1-repeat"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFirst, + }); + const repeatedEvent = ( + Array.isArray(repeated) ? repeated[0] : repeated + ) as PlannedThreadDeletedEvent; + expect(repeatedEvent.type).toBe("thread.deleted"); + if (repeatedEvent.type !== "thread.deleted") return; + expect(repeatedEvent.payload.worktreeCleanup).toEqual(firstEvent.payload.worktreeCleanup); + const afterRepeat = yield* projectEvent(afterFirst, { ...repeatedEvent, sequence: 5 }); + expect( + afterRepeat.threads.find((thread) => thread.id === asThreadId("thread-delete-1")) + ?.worktreeCleanup, + ).toEqual(firstEvent.payload.worktreeCleanup); + + const second = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-2"), + threadId: asThreadId("thread-delete-2"), + deleteWorktree: true, + }, + readModel: afterFirst, + }); + const secondEvent = (Array.isArray(second) ? second[0] : second) as PlannedThreadDeletedEvent; + expect(secondEvent.type).toBe("thread.deleted"); + if (secondEvent.type !== "thread.deleted") return; + expect(secondEvent.payload.worktreeCleanup).toMatchObject({ + status: "queued", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-2", + blockedByThreadId: asThreadId("thread-delete-1"), + }); + }), + ); + + it.effect("queues cleanups from different checkouts that share a Git common directory", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const firstReadModel = { + ...seeded, + threads: seeded.threads.map((thread, index) => ({ + ...thread, + projectId: index === 1 ? asProjectId("project-delete-sibling") : thread.projectId, + branch: `sibling-cleanup-${index + 1}`, + worktreePath: `/tmp/sibling-worktrees/cleanup-${index + 1}`, + })), + projects: [ + ...seeded.projects, + { + ...seeded.projects[0]!, + id: asProjectId("project-delete-sibling"), + workspaceRoot: "/tmp/project-delete-sibling", + }, + ], + }; + + const first = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-sibling-delete-1"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + repositoryKey: "/tmp/shared-repository/.git", + }, + readModel: firstReadModel, + }); + const firstEvent = (Array.isArray(first) ? first[0] : first) as PlannedThreadDeletedEvent; + const afterFirst = yield* projectEvent(firstReadModel, { ...firstEvent, sequence: 4 }); + + const second = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-sibling-delete-2"), + threadId: asThreadId("thread-delete-2"), + deleteWorktree: true, + repositoryKey: "/tmp/shared-repository/.git", + }, + readModel: afterFirst, + }); + const secondEvent = (Array.isArray(second) ? second[0] : second) as PlannedThreadDeletedEvent; + + expect(firstEvent.payload.worktreeCleanup).toMatchObject({ + status: "deleting", + repositoryRoot: "/tmp/project-delete", + repositoryKey: "/tmp/shared-repository/.git", + }); + expect(secondEvent.payload.worktreeCleanup).toMatchObject({ + status: "queued", + repositoryRoot: "/tmp/project-delete-sibling", + repositoryKey: "/tmp/shared-repository/.git", + blockedByThreadId: asThreadId("thread-delete-1"), + }); + }), + ); + + it.effect("rejects deleting a worktree registered as an active project root", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const worktreePath = "/tmp/project-delete-worktrees/active-project"; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { ...thread, branch: "active-project", worktreePath } + : thread, + ), + projects: [ + ...seeded.projects, + { + ...seeded.projects[0]!, + id: asProjectId("project-active-worktree"), + workspaceRoot: worktreePath, + }, + ], + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-delete-active-project-worktree"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }), + ); + + expect(error.message).toContain("project-active-worktree"); + expect(error.message).toContain("workspace root"); + }), + ); + + it.effect("refuses to delete a worktree still owned by another live thread", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => ({ + ...thread, + branch: "shared-cleanup", + worktreePath: "/tmp/project-delete-worktrees/shared-cleanup", + })), + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-shared-worktree"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }), + ); + expect(error.message).toContain("is still used by thread 'thread-delete-2'"); + }), + ); + + it.effect("retries or abandons a persisted cleanup failure", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { + ...thread, + branch: "cleanup-retry", + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + } + : thread, + ), + }; + const deleted = (yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-cleanup-retry-delete"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + })) as PlannedThreadDeletedEvent; + const afterDelete = yield* projectEvent(readModel, { ...deleted, sequence: 4 }); + + const earlyAbandonError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.abandon", + commandId: asCommandId("cmd-cleanup-abandon-early"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterDelete, + }), + ); + expect(earlyAbandonError.message).toContain("does not have failed worktree cleanup"); + + const pathReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.meta.update", + commandId: asCommandId("cmd-cleanup-path-reuse"), + threadId: asThreadId("thread-delete-2"), + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + }, + readModel: afterDelete, + }), + ); + expect(pathReuseError.message).toContain("is still being cleaned up by thread"); + + const projectCreateReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.create", + commandId: asCommandId("cmd-cleanup-project-create-reuse"), + projectId: asProjectId("project-cleanup-reuse"), + title: "Cleanup reuse", + workspaceRoot: "/tmp/project-delete-worktrees/cleanup-retry", + createdAt: "2026-01-01T00:00:00.000Z", + }, + readModel: afterDelete, + }), + ); + expect(projectCreateReuseError.message).toContain( + "is still being cleaned up by thread 'thread-delete-1'", + ); + + const projectUpdateReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.meta.update", + commandId: asCommandId("cmd-cleanup-project-update-reuse"), + projectId: asProjectId("project-delete"), + workspaceRoot: "/tmp/project-delete-worktrees/cleanup-retry", + }, + readModel: afterDelete, + }), + ); + expect(projectUpdateReuseError.message).toContain( + "is still being cleaned up by thread 'thread-delete-1'", + ); + + const failed = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.update", + commandId: asCommandId("cmd-cleanup-failed"), + threadId: asThreadId("thread-delete-1"), + cleanup: { + status: "failed", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + startedAt: "2026-01-01T00:00:00.000Z", + failedAt: "2026-01-01T00:00:01.000Z", + error: "permission denied", + }, + }, + readModel: afterDelete, + }); + const failedEvent = (Array.isArray(failed) ? failed[0] : failed) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + const afterFailure = yield* projectEvent(afterDelete, { ...failedEvent, sequence: 5 }); + + const retry = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.retry", + commandId: asCommandId("cmd-cleanup-retry"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFailure, + }); + const retryEvent = (Array.isArray(retry) ? retry[0] : retry) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + expect(retryEvent.payload.cleanup?.status).toBe("deleting"); + + const abandon = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.abandon", + commandId: asCommandId("cmd-cleanup-abandon"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFailure, + }); + const abandonEvent = (Array.isArray(abandon) ? abandon[0] : abandon) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + expect(abandonEvent.payload.cleanup).toBeNull(); + }), + ); + it.effect("rejects deleting a non-empty project without force", () => Effect.gen(function* () { const readModel = yield* seedReadModel; @@ -154,6 +490,44 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }), ); + it.effect("rejects project deletion while a deleted thread is cleaning up its worktree", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { + ...thread, + deletedAt: "2026-01-01T00:00:01.000Z", + worktreeCleanup: { + status: "deleting" as const, + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-1", + startedAt: "2026-01-01T00:00:01.000Z", + }, + } + : { ...thread, deletedAt: "2026-01-01T00:00:01.000Z" }, + ), + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.delete", + commandId: asCommandId("cmd-project-delete-during-cleanup"), + projectId: asProjectId("project-delete"), + force: true, + }, + readModel, + }), + ); + + expect(error.message).toContain("thread-delete-1"); + expect(error.message).toContain("Wait for cleanup to finish or keep the worktree first"); + }), + ); + it.effect("reuses thread.delete semantics when force-deleting a non-empty project", () => Effect.gen(function* () { const readModel = yield* seedReadModel; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 96016f702421..aeb3cebddddf 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -20,6 +20,7 @@ import { requireThreadAbsent, requireThreadNotArchived, } from "./commandInvariants.ts"; +import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; import { projectEvent } from "./projector.ts"; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); @@ -158,6 +159,62 @@ function nextAnnotationAnchorMessageId(thread: OrchestrationReadModel["threads"] return latestUserMessageId(thread) ?? thread.annotation?.anchorMessageId; } +function worktreeCleanupTimestamp( + cleanup: NonNullable, +): string { + switch (cleanup.status) { + case "deleting": + return cleanup.startedAt; + case "queued": + return cleanup.queuedAt; + case "failed": + return cleanup.failedAt; + } +} + +function findWorktreeCleanupBlocker( + readModel: OrchestrationReadModel, + repositoryKey: string, + exceptThreadId?: string, +) { + const normalizedKey = normalizeProjectPathForComparison(repositoryKey); + return readModel.threads + .filter((candidate) => { + const cleanup = candidate.worktreeCleanup; + return ( + candidate.id !== exceptThreadId && + cleanup != null && + cleanup.status !== "failed" && + normalizeProjectPathForComparison(cleanup.repositoryKey ?? cleanup.repositoryRoot) === + normalizedKey + ); + }) + .toSorted((left, right) => { + const leftCleanup = left.worktreeCleanup; + const rightCleanup = right.worktreeCleanup; + if (leftCleanup == null || rightCleanup == null) return 0; + return ( + worktreeCleanupTimestamp(rightCleanup).localeCompare( + worktreeCleanupTimestamp(leftCleanup), + ) || right.id.localeCompare(left.id) + ); + })[0]; +} + +function findWorktreeCleanupOwner( + readModel: OrchestrationReadModel, + worktreePath: string, + exceptThreadId?: string, +) { + const normalizedPath = normalizeProjectPathForComparison(worktreePath); + return readModel.threads.find( + (candidate) => + candidate.id !== exceptThreadId && + candidate.worktreeCleanup != null && + normalizeProjectPathForComparison(candidate.worktreeCleanup.worktreePath) === normalizedPath, + ); +} + function withEventBase( input: Pick & { readonly aggregateKind: OrchestrationEvent["aggregateKind"]; @@ -252,6 +309,13 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" workspaceRoot: command.workspaceRoot, exceptProjectId: command.projectId, }); + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.workspaceRoot); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Workspace root '${command.workspaceRoot}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } return { ...(yield* withEventBase({ @@ -287,6 +351,13 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" workspaceRoot: command.workspaceRoot, exceptProjectId: command.projectId, }); + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.workspaceRoot); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Workspace root '${command.workspaceRoot}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } } const occurredAt = yield* nowIso; return { @@ -320,9 +391,15 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" command, projectId: command.projectId, }); - const activeThreads = listThreadsByProjectId(readModel, command.projectId).filter( - (thread) => thread.deletedAt === null, - ); + const projectThreads = listThreadsByProjectId(readModel, command.projectId); + const cleanupThread = projectThreads.find((thread) => thread.worktreeCleanup != null); + if (cleanupThread !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Project '${command.projectId}' cannot be deleted while thread '${cleanupThread.id}' is cleaning up its worktree. Wait for cleanup to finish or keep the worktree first.`, + }); + } + const activeThreads = projectThreads.filter((thread) => thread.deletedAt === null); if (activeThreads.length > 0 && command.force !== true) { return yield* new OrchestrationCommandInvariantError({ commandType: command.type, @@ -338,6 +415,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" type: "thread.delete", commandId: command.commandId, threadId: thread.id, + ...(command.repositoryKey === undefined + ? {} + : { repositoryKey: command.repositoryKey }), }), ), { @@ -376,6 +456,15 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" command, threadId: command.threadId, }); + if (command.worktreePath !== null) { + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.worktreePath); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${command.worktreePath}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } + } return { ...(yield* withEventBase({ aggregateKind: "thread", @@ -400,12 +489,98 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" } case "thread.delete": { - yield* requireThread({ + const thread = yield* requireThread({ readModel, command, threadId: command.threadId, }); const occurredAt = yield* nowIso; + + // Deletion commands can be retried after the first deleted event has + // already been projected. Preserve the tombstone, especially its + // durable worktree-cleanup state, rather than allowing a retry that + // omits deleteWorktree to clear an in-flight cleanup. + if (thread.deletedAt !== null) { + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt, + commandId: command.commandId, + })), + type: "thread.deleted", + payload: { + threadId: command.threadId, + deletedAt: thread.deletedAt, + ...(thread.worktreeCleanup === null ? {} : { worktreeCleanup: thread.worktreeCleanup }), + }, + }; + } + + let worktreeCleanup: NonNullable< + OrchestrationReadModel["threads"][number]["worktreeCleanup"] + > | null = null; + if (command.deleteWorktree === true) { + if (thread.worktreePath === null) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not own a worktree to delete.`, + }); + } + const project = yield* requireProject({ + readModel, + command, + projectId: thread.projectId, + }); + const normalizedWorktreePath = normalizeProjectPathForComparison(thread.worktreePath); + const sharedProject = readModel.projects.find( + (candidate) => + candidate.deletedAt === null && + normalizeProjectPathForComparison(candidate.workspaceRoot) === normalizedWorktreePath, + ); + if (sharedProject !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${thread.worktreePath}' is still used as the workspace root of project '${sharedProject.id}'.`, + }); + } + const sharedThread = readModel.threads.find( + (candidate) => + candidate.id !== thread.id && + candidate.deletedAt === null && + candidate.worktreePath !== null && + normalizeProjectPathForComparison(candidate.worktreePath) === normalizedWorktreePath, + ); + if (sharedThread !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${thread.worktreePath}' is still used by thread '${sharedThread.id}'.`, + }); + } + const repositoryKey = command.repositoryKey; + const blocker = findWorktreeCleanupBlocker( + readModel, + repositoryKey ?? project.workspaceRoot, + thread.id, + ); + worktreeCleanup = + blocker === undefined + ? { + status: "deleting", + repositoryRoot: project.workspaceRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), + worktreePath: thread.worktreePath, + startedAt: occurredAt, + } + : { + status: "queued", + repositoryRoot: project.workspaceRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), + worktreePath: thread.worktreePath, + queuedAt: occurredAt, + blockedByThreadId: blocker.id, + }; + } return { ...(yield* withEventBase({ aggregateKind: "thread", @@ -417,10 +592,126 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" payload: { threadId: command.threadId, deletedAt: occurredAt, + ...(worktreeCleanup === null ? {} : { worktreeCleanup }), }, }; } + case "thread.worktree-cleanup.retry": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + const cleanup = thread.worktreeCleanup; + if (thread.deletedAt === null || cleanup == null || cleanup.status !== "failed") { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not have a failed worktree cleanup to retry.`, + }); + } + const occurredAt = yield* nowIso; + const repositoryKey = cleanup.repositoryKey; + const blocker = findWorktreeCleanupBlocker( + readModel, + repositoryKey ?? cleanup.repositoryRoot, + thread.id, + ); + const nextCleanup = + blocker === undefined + ? { + status: "deleting" as const, + repositoryRoot: cleanup.repositoryRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), + worktreePath: cleanup.worktreePath, + startedAt: occurredAt, + } + : { + status: "queued" as const, + repositoryRoot: cleanup.repositoryRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), + worktreePath: cleanup.worktreePath, + queuedAt: occurredAt, + blockedByThreadId: blocker.id, + }; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: nextCleanup, updatedAt: occurredAt }, + }; + } + + case "thread.worktree-cleanup.abandon": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + if ( + thread.deletedAt === null || + thread.worktreeCleanup == null || + thread.worktreeCleanup.status !== "failed" + ) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not have failed worktree cleanup to abandon.`, + }); + } + const occurredAt = yield* nowIso; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: null, updatedAt: occurredAt }, + }; + } + + case "thread.worktree-cleanup.update": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + const current = thread.worktreeCleanup; + if (thread.deletedAt === null || current == null) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' no longer has active worktree cleanup.`, + }); + } + if ( + command.cleanup !== null && + (command.cleanup.repositoryRoot !== current.repositoryRoot || + command.cleanup.worktreePath !== current.worktreePath) + ) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' cleanup paths cannot change during processing.`, + }); + } + const validTransition = + (current.status === "queued" && + (command.cleanup?.status === "deleting" || command.cleanup?.status === "failed")) || + (current.status === "deleting" && + (command.cleanup === null || + command.cleanup.status === "deleting" || + command.cleanup.status === "failed")); + if (!validTransition) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Invalid worktree cleanup transition from '${current.status}' to '${command.cleanup?.status ?? "complete"}'.`, + }); + } + const occurredAt = yield* nowIso; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: command.cleanup, updatedAt: occurredAt }, + }; + } + case "thread.archive": { yield* requireThreadNotArchived({ readModel, @@ -948,6 +1239,19 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" thread.branch !== command.expectedBranch ? thread.branch : command.branch; + if (command.worktreePath != null) { + const cleanupOwner = findWorktreeCleanupOwner( + readModel, + command.worktreePath, + command.threadId, + ); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${command.worktreePath}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } + } const occurredAt = yield* nowIso; return { ...(yield* withEventBase({ diff --git a/apps/server/src/orchestration/projector.test.ts b/apps/server/src/orchestration/projector.test.ts index 9a2f6c55a75d..f2d71f15441c 100644 --- a/apps/server/src/orchestration/projector.test.ts +++ b/apps/server/src/orchestration/projector.test.ts @@ -86,6 +86,7 @@ describe("orchestration projector", () => { interactionMode: "default", branch: null, worktreePath: null, + worktreeCleanup: null, latestTurn: null, createdAt: now, updatedAt: now, diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index 743c11138cf5..5c2e91255b13 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -23,6 +23,7 @@ import { ThreadArchivedPayload, ThreadCreatedPayload, ThreadDeletedPayload, + ThreadWorktreeCleanupUpdatedPayload, ThreadInteractionModeSetPayload, ThreadMetaUpdatedPayload, ThreadProposedPlanUpsertedPayload, @@ -323,6 +324,7 @@ export function projectEvent( snoozedUntil: null, snoozedAt: null, annotation: null, + worktreeCleanup: null, deletedAt: null, messages: [], activities: [], @@ -347,11 +349,28 @@ export function projectEvent( ...nextBase, threads: updateThread(nextBase.threads, payload.threadId, { deletedAt: payload.deletedAt, + worktreeCleanup: payload.worktreeCleanup ?? null, updatedAt: payload.deletedAt, }), })), ); + case "thread.worktree-cleanup-updated": + return decodeForEvent( + ThreadWorktreeCleanupUpdatedPayload, + event.payload, + event.type, + "payload", + ).pipe( + Effect.map((payload) => ({ + ...nextBase, + threads: updateThread(nextBase.threads, payload.threadId, { + worktreeCleanup: payload.cleanup, + updatedAt: payload.updatedAt, + }), + })), + ); + case "thread.archived": return decodeForEvent(ThreadArchivedPayload, event.payload, event.type, "payload").pipe( Effect.map((payload) => ({ diff --git a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts index 128430c13eb0..a326e5367e68 100644 --- a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts +++ b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts @@ -98,7 +98,7 @@ projectionRepositoriesLayer("Projection repositories", (it) => { runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: null, + worktreePath: "/tmp/thread-null-options-worktree", latestTurnId: null, createdAt: "2026-03-24T00:00:00.000Z", updatedAt: "2026-03-24T00:00:00.000Z", @@ -167,6 +167,10 @@ projectionRepositoriesLayer("Projection repositories", (it) => { Option.getOrNull(persisted)?.latestUserMessageId, MessageId.make("message-1"), ); + assert.deepStrictEqual( + (yield* threads.listActiveWorktreeOwners()).map((thread) => thread.threadId), + [ThreadId.make("thread-null-options")], + ); }), ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index ca72c8537337..6058acdfc152 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -7,19 +7,23 @@ import * as Struct from "effect/Struct"; import { toPersistenceSqlError } from "../Errors.ts"; import { + ActiveWorktreeOwner, DeleteProjectionThreadInput, GetProjectionThreadInput, + ListActiveWorktreeOwnerThreadsInput, ListProjectionThreadsByProjectInput, + ListPendingWorktreeCleanupThreadsInput, ProjectionThread, ProjectionThreadRepository, type ProjectionThreadRepositoryShape, } from "../Services/ProjectionThreads.ts"; -import { ModelSelection, ThreadAnnotation } from "@t3tools/contracts"; +import { ModelSelection, ThreadAnnotation, ThreadWorktreeCleanup } from "@t3tools/contracts"; const ProjectionThreadDbRow = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), annotation: Schema.NullOr(Schema.fromJsonString(ThreadAnnotation)), + worktreeCleanup: Schema.NullOr(Schema.fromJsonString(ThreadWorktreeCleanup)), }), ); type ProjectionThreadDbRow = typeof ProjectionThreadDbRow.Type; @@ -53,6 +57,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id, title_regeneration_started_at, annotation_json, + worktree_cleanup_json, latest_user_message_id, latest_user_message_at, pending_approval_count, @@ -82,6 +87,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { ${row.titleRegenerationRequestId ?? null}, ${row.titleRegenerationStartedAt ?? null}, ${row.annotation === null ? null : JSON.stringify(row.annotation)}, + ${row.worktreeCleanup == null ? null : JSON.stringify(row.worktreeCleanup)}, ${row.latestUserMessageId}, ${row.latestUserMessageAt}, ${row.pendingApprovalCount}, @@ -111,6 +117,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id = excluded.title_regeneration_request_id, title_regeneration_started_at = excluded.title_regeneration_started_at, annotation_json = excluded.annotation_json, + worktree_cleanup_json = excluded.worktree_cleanup_json, latest_user_message_id = excluded.latest_user_message_id, latest_user_message_at = excluded.latest_user_message_at, pending_approval_count = excluded.pending_approval_count, @@ -147,6 +154,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -185,6 +193,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -206,6 +215,62 @@ const makeProjectionThreadRepository = Effect.gen(function* () { `, }); + const listPendingWorktreeCleanupRows = SqlSchema.findAll({ + Request: ListPendingWorktreeCleanupThreadsInput, + Result: ProjectionThreadDbRow, + execute: () => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", + latest_user_message_id AS "latestUserMessageId", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE worktree_cleanup_json IS NOT NULL + AND json_extract(worktree_cleanup_json, '$.status') IN ('deleting', 'queued') + ORDER BY deleted_at ASC, thread_id ASC + `, + }); + + const listActiveWorktreeOwnerRows = SqlSchema.findAll({ + Request: ListActiveWorktreeOwnerThreadsInput, + Result: ActiveWorktreeOwner, + execute: () => + sql` + SELECT + thread_id AS "threadId", + worktree_path AS "worktreePath" + FROM projection_threads + WHERE deleted_at IS NULL + AND worktree_path IS NOT NULL + ORDER BY created_at ASC, thread_id ASC + `, + }); + const upsert: ProjectionThreadRepositoryShape["upsert"] = (row) => upsertProjectionThreadRow(row).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.upsert:query")), @@ -221,6 +286,22 @@ const makeProjectionThreadRepository = Effect.gen(function* () { Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.listByProjectId:query")), ); + const listPendingWorktreeCleanup: ProjectionThreadRepositoryShape["listPendingWorktreeCleanup"] = + () => + listPendingWorktreeCleanupRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadRepository.listPendingWorktreeCleanup:query"), + ), + ); + + const listActiveWorktreeOwners: ProjectionThreadRepositoryShape["listActiveWorktreeOwners"] = + () => + listActiveWorktreeOwnerRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadRepository.listActiveWorktreeOwners:query"), + ), + ); + const deleteById: ProjectionThreadRepositoryShape["deleteById"] = (input) => deleteProjectionThreadRow(input).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.deleteById:query")), @@ -230,6 +311,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { upsert, getById, listByProjectId, + listPendingWorktreeCleanup, + listActiveWorktreeOwners, deleteById, } satisfies ProjectionThreadRepositoryShape; }); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 96d92057bb29..4cf2d211a2e0 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -58,6 +58,7 @@ import Migration0042 from "./Migrations/042_ProjectionThreadAnnotation.ts"; import Migration0043 from "./Migrations/043_UpdateDrain.ts"; import Migration0044 from "./Migrations/044_UpdateDrainClaim.ts"; import Migration0045 from "./Migrations/045_ProjectionTurnRequestCorrelations.ts"; +import Migration0046 from "./Migrations/046_ProjectionThreadWorktreeCleanup.ts"; /** * Migration loader with all migrations defined inline. @@ -115,6 +116,7 @@ export const migrationEntries = [ [43, "UpdateDrain", Migration0043], [44, "UpdateDrainClaim", Migration0044], [45, "ProjectionTurnRequestCorrelations", Migration0045], + [46, "ProjectionThreadWorktreeCleanup", Migration0046], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts new file mode 100644 index 000000000000..765a9605684b --- /dev/null +++ b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts @@ -0,0 +1,55 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("046_ProjectionThreadWorktreeCleanup", (it) => { + it.effect("adds nullable cleanup state without changing existing rows", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 44 }); + yield* sql` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + created_at, + updated_at + ) + VALUES ( + 'thread-before-cleanup', + 'project-1', + 'Existing thread', + '{"instanceId":"codex","model":"gpt-5-codex"}', + 'full-access', + 'default', + '2026-08-23T00:00:00.000Z', + '2026-08-23T00:00:00.000Z' + ) + `; + + yield* runMigrations({ toMigrationInclusive: 46 }); + + const columns = yield* sql<{ readonly name: string; readonly notnull: number }>` + PRAGMA table_info(projection_threads) + `; + const cleanupJson = columns.find((column) => column.name === "worktree_cleanup_json"); + assert.equal(cleanupJson?.notnull, 0); + + const rows = yield* sql<{ readonly cleanup: string | null }>` + SELECT worktree_cleanup_json AS cleanup + FROM projection_threads + WHERE thread_id = 'thread-before-cleanup' + `; + assert.equal(rows[0]?.cleanup, null); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts new file mode 100644 index 000000000000..5926e05b1e47 --- /dev/null +++ b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts @@ -0,0 +1,16 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + + if (!columns.some((column) => column.name === "worktree_cleanup_json")) { + yield* sql` + ALTER TABLE projection_threads + ADD COLUMN worktree_cleanup_json TEXT + `; + } +}); diff --git a/apps/server/src/persistence/Services/ProjectionThreads.ts b/apps/server/src/persistence/Services/ProjectionThreads.ts index b5f450c1841a..5d3d9eed2d0f 100644 --- a/apps/server/src/persistence/Services/ProjectionThreads.ts +++ b/apps/server/src/persistence/Services/ProjectionThreads.ts @@ -16,6 +16,7 @@ import { ProviderInteractionMode, RuntimeMode, ThreadAnnotation, + ThreadWorktreeCleanup, ThreadId, TurnId, } from "@t3tools/contracts"; @@ -48,6 +49,7 @@ export const ProjectionThread = Schema.Struct({ titleRegenerationRequestId: Schema.optional(Schema.NullOr(CommandId)), titleRegenerationStartedAt: Schema.optional(Schema.NullOr(IsoDateTime)), annotation: Schema.NullOr(ThreadAnnotation), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), latestUserMessageId: Schema.NullOr(MessageId), latestUserMessageAt: Schema.NullOr(IsoDateTime), pendingApprovalCount: NonNegativeInt, @@ -70,6 +72,13 @@ export type DeleteProjectionThreadInput = typeof DeleteProjectionThreadInput.Typ export const ListProjectionThreadsByProjectInput = Schema.Struct({ projectId: ProjectId, }); +export const ListPendingWorktreeCleanupThreadsInput = Schema.Void; +export const ListActiveWorktreeOwnerThreadsInput = Schema.Void; +export const ActiveWorktreeOwner = Schema.Struct({ + threadId: ThreadId, + worktreePath: Schema.String, +}); +export type ActiveWorktreeOwner = typeof ActiveWorktreeOwner.Type; export type ListProjectionThreadsByProjectInput = typeof ListProjectionThreadsByProjectInput.Type; /** @@ -99,6 +108,16 @@ export interface ProjectionThreadRepositoryShape { input: ListProjectionThreadsByProjectInput, ) => Effect.Effect, ProjectionRepositoryError>; + readonly listPendingWorktreeCleanup: () => Effect.Effect< + ReadonlyArray, + ProjectionRepositoryError + >; + + readonly listActiveWorktreeOwners: () => Effect.Effect< + ReadonlyArray, + ProjectionRepositoryError + >; + /** * Soft-delete a projected thread row by id. */ diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index d6688d369692..562722b156c6 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -758,6 +758,50 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => { assert.notInclude(error.detail, "Git command failed in"); }), ); + + it.effect("allows an unregistered missing worktree when explicitly requested", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const pathService = yield* Path.Path; + const missingWorktree = pathService.join(cwd, "missing-worktree"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.initRepo({ cwd }); + + yield* driver.removeWorktree({ + cwd, + path: missingWorktree, + force: true, + allowMissing: true, + }); + }), + ); + + it.effect("does not treat an absent locked worktree as removed", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const worktreePath = pathService.join(yield* makeTmpDir("git-worktrees-"), "locked"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/locked-worktree", + }); + yield* git(cwd, ["worktree", "lock", worktreePath]); + yield* fileSystem.remove(worktreePath, { recursive: true }); + + const error = yield* driver + .removeWorktree({ cwd, path: worktreePath, force: true, allowMissing: true }) + .pipe(Effect.flip); + const registered = yield* git(cwd, ["worktree", "list", "--porcelain"]); + + assert.equal(error._tag, "GitCommandError"); + assert.include(registered, worktreePath); + }), + ); }); describe("review diff previews", () => { diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 97e3f8dc9265..7119174a40b3 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -2991,6 +2991,29 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* args.push("--force"); } args.push(input.path); + if (input.allowMissing === true) { + const result = yield* executeGitWithStableDiagnostics( + "GitVcsDriver.removeWorktree", + input.cwd, + args, + { + allowNonZeroExit: true, + timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, + }, + ); + if (result.exitCode === 0 || result.stderr.includes("is not a working tree")) return; + return yield* new GitCommandError({ + ...gitCommandContext({ + operation: "GitVcsDriver.removeWorktree", + cwd: input.cwd, + args, + }), + detail: "git worktree remove failed", + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }); + } yield* executeGit("GitVcsDriver.removeWorktree", input.cwd, args, { timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, fallbackErrorDetail: "git worktree remove failed", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b523600b47eb..ad753f5c4ad0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -637,7 +637,6 @@ const makeWsRpcLayer = ( projectId: event.payload.projectId, }), ); - case "thread.deleted": case "thread.archived": return Effect.succeed( Option.some({ @@ -646,6 +645,8 @@ const makeWsRpcLayer = ( threadId: event.payload.threadId, }), ); + case "thread.deleted": + return threadUpsertOrRemove(event.payload.threadId, event.sequence); case "thread.unarchived": return threadUpsertOrRemove(event.payload.threadId, event.sequence); default: diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index 8e37ca7e57db..55bbcea60be5 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -224,7 +224,11 @@ import { type ProviderInstanceEntry, } from "../providerInstances"; import { getTriggerDisplayModelLabel } from "./chat/providerIconUtils"; -import { SidebarThreadHoverContent } from "./sidebar/SidebarThreadHoverContent"; +import { + SidebarThreadCleanupHoverContent, + SidebarThreadHoverContent, +} from "./sidebar/SidebarThreadHoverContent"; +import { WorktreeCleanupFailureDialog } from "./WorktreeCleanupFailureDialog"; import { buildPhysicalToLogicalProjectKeyMap, buildSidebarProjectSnapshots, @@ -407,6 +411,10 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr } = props; const threadRef = scopeThreadRef(thread.environmentId, thread.id); const threadKey = scopedThreadKey(threadRef); + const cleanup = thread.worktreeCleanup ?? null; + const isCleanupPending = cleanup?.status === "deleting" || cleanup?.status === "queued"; + const isCleanupFailed = cleanup?.status === "failed"; + const [cleanupFailureOpen, setCleanupFailureOpen] = useState(false); const lastVisitedAt = useUiStateStore((state) => state.threadLastVisitedAtById[threadKey]); const isSelected = useThreadSelectionStore((state) => state.selectedThreadKeys.has(threadKey)); const runningTerminalIds = useThreadRunningTerminalIds({ @@ -498,6 +506,11 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr const terminalStatus = terminalStatusFromRunningIds(runningTerminalIds); const isConfirmingArchive = confirmingArchiveThreadKey === threadKey && !isThreadRunning; const hasActiveAnnotation = thread.annotation?.resolvedAt === null; + const cleanupBlockerTitle = + cleanup?.status === "queued" + ? (readThreadShell(scopeThreadRef(thread.environmentId, cleanup.blockedByThreadId))?.title ?? + null) + : null; const branchMismatch = resolveLocalCheckoutBranchMismatch({ effectiveEnvMode: thread.worktreePath === null ? "local" : "worktree", activeWorktreePath: thread.worktreePath, @@ -530,6 +543,15 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr terminalProcessCount={runningTerminalIds.length} terminalStatus={terminalStatus} thread={thread} + cleanupBlockerTitle={cleanupBlockerTitle} + showCleanup={!hasActiveAnnotation} + /> + ); + const cleanupHoverDetails = ( + ); const threadMetaClassName = isConfirmingArchive @@ -565,12 +587,22 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr ); const handleRowClick = useCallback( (event: React.MouseEvent) => { + if (isCleanupFailed) { + event.preventDefault(); + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) { + event.preventDefault(); + return; + } handleThreadClick(event, threadRef, orderedProjectThreadKeys); }, - [handleThreadClick, orderedProjectThreadKeys, threadRef], + [handleThreadClick, isCleanupFailed, isCleanupPending, orderedProjectThreadKeys, threadRef], ); const handleRowDoubleClick = useCallback( (event: React.MouseEvent) => { + if (cleanup !== null) return; // Already renaming this row: a double-click on the row chrome (outside the // input) must not restart and discard the in-progress edit. if (renamingThreadKey === threadKey) return; @@ -585,19 +617,25 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr event.preventDefault(); startThreadRename(threadKey, thread.title); }, - [isMobile, renamingThreadKey, startThreadRename, threadKey, thread.title], + [cleanup, isMobile, renamingThreadKey, startThreadRename, threadKey, thread.title], ); const handleRowKeyDown = useCallback( (event: React.KeyboardEvent) => { if (event.key !== "Enter" && event.key !== " ") return; event.preventDefault(); + if (isCleanupFailed) { + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) return; navigateToThread(threadRef); }, - [navigateToThread, threadRef], + [isCleanupFailed, isCleanupPending, navigateToThread, threadRef], ); const handleRowContextMenu = useCallback( (event: React.MouseEvent) => { event.preventDefault(); + if (cleanup !== null) return; const hasSelection = useThreadSelectionStore.getState().hasSelection(); if (hasSelection && isSelected) { void (async () => { @@ -643,7 +681,14 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr } })(); }, - [clearSelection, handleMultiSelectContextMenu, handleThreadContextMenu, isSelected, threadRef], + [ + cleanup, + clearSelection, + handleMultiSelectContextMenu, + handleThreadContextMenu, + isSelected, + threadRef, + ], ); const handlePrClick = useCallback( (event: React.MouseEvent) => { @@ -770,17 +815,39 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr size="sm" isActive={isActive} data-testid={`thread-row-${thread.id}`} + aria-disabled={isCleanupPending || undefined} className={`${resolveThreadRowClassName({ isActive, isSelected, - })} relative isolate`} + })} relative isolate ${isCleanupPending ? "cursor-not-allowed opacity-65" : ""}`} onClick={handleRowClick} onDoubleClick={handleRowDoubleClick} onKeyDown={handleRowKeyDown} onContextMenu={handleRowContextMenu} > + {isCleanupPending && !hasActiveAnnotation ? ( + + + } + /> + + {threadHoverDetails} + + + ) : null}
- {prStatus && ( + {cleanup === null && prStatus && (
- {discoveredPorts.length > 0 && ( + {cleanup === null && discoveredPorts.length > 0 && ( Confirm - ) : !isThreadRunning ? ( + ) : !isThreadRunning && cleanup === null ? ( appSettingsConfirmThreadArchive ? (
+ + + + + + ); +} diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 8397e8904b90..7d22de14b502 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -7,10 +7,10 @@ import { type BackgroundActivityProfile, type DesktopUpdateChannel, ProviderDriverKind, - type ScopedThreadRef, type SidebarProjectGroupingMode, } from "@t3tools/contracts"; import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; import { isAtomCommandInterrupted, settlePromise, @@ -2516,7 +2516,8 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null ); const handleArchivedThreadContextMenu = useCallback( - async (threadRef: ScopedThreadRef, position: { x: number; y: number }) => { + async (thread: EnvironmentThreadShell, position: { x: number; y: number }) => { + const threadRef = scopeThreadRef(thread.environmentId, thread.id); const api = readLocalApi(); if (!api) return; const clicked = await api.contextMenu.show( @@ -2545,7 +2546,10 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null } if (clicked === "delete") { - const result = await confirmAndDeleteThread(threadRef); + const archivedThreads = archivedGroups + .filter((group) => group.project.environmentId === thread.environmentId) + .flatMap((group) => group.threads); + const result = await confirmAndDeleteThread(threadRef, { archivedThreads }); if (result._tag === "Success") { refreshArchivedThreads(); } else if (!isAtomCommandInterrupted(result)) { @@ -2560,7 +2564,7 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null } } }, - [confirmAndDeleteThread, refreshArchivedThreads, unarchiveThread], + [archivedGroups, confirmAndDeleteThread, refreshArchivedThreads, unarchiveThread], ); return ( @@ -2618,13 +2622,10 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null event.preventDefault(); void (async () => { const result = await settlePromise(() => - handleArchivedThreadContextMenu( - scopeThreadRef(thread.environmentId, thread.id), - { - x: event.clientX, - y: event.clientY, - }, - ), + handleArchivedThreadContextMenu(thread, { + x: event.clientX, + y: event.clientY, + }), ); if (result._tag === "Failure") { const error = squashAtomCommandFailure(result); diff --git a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx index 4826863b5ac5..dbf259ee4031 100644 --- a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx +++ b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx @@ -23,6 +23,8 @@ export interface SidebarThreadHoverContentProps { } | null; terminalStatus: TerminalStatusIndicator | null; terminalProcessCount: number; + cleanupBlockerTitle?: string | null; + showCleanup?: boolean; } function terminalProcessLabel(count: number): string { @@ -109,6 +111,48 @@ export function SidebarThreadHoverContent(props: SidebarThreadHoverContentProps)
) : null}
+ {props.showCleanup === false ? null : ( + + )} +
+ ); +} + +export function SidebarThreadCleanupHoverContent(props: { + thread: SidebarThreadSummary; + blockerTitle: string | null; + standalone?: boolean; +}) { + const cleanup = props.thread.worktreeCleanup; + if (cleanup == null || cleanup.status === "failed") return null; + + return ( +
+ {cleanup.status === "deleting" ? ( + <> +
Deleting worktree
+
+ {cleanup.worktreePath} +
+ + ) : ( + <> +
Waiting for cleanup
+
+ {cleanup.blockedByThreadId} + {props.blockerTitle ? ` — ${props.blockerTitle}` : ""} +
+ + )}
); } diff --git a/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx b/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx index e9ded512f939..083b75d03896 100644 --- a/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx +++ b/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx @@ -268,6 +268,7 @@ export function ThreadAnnotationHoverPopover(props: { rowActive: boolean; trigger: ReactNode; threadDetails: ReactNode; + trailingContent?: ReactNode; onEdit: () => void; onResolve: () => void; onBodyChange: (body: string) => Promise; @@ -379,6 +380,7 @@ export function ThreadAnnotationHoverPopover(props: { /> + {props.trailingContent} diff --git a/apps/web/src/hooks/useThreadActions.test.ts b/apps/web/src/hooks/useThreadActions.test.ts index c5385211591f..764fc622d726 100644 --- a/apps/web/src/hooks/useThreadActions.test.ts +++ b/apps/web/src/hooks/useThreadActions.test.ts @@ -1,7 +1,14 @@ import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { ThreadArchiveBlockedError } from "./useThreadActions"; +import { getOrphanedWorktreePathForThread } from "../worktreeCleanup"; +import { + collectThreadDeleteCandidates, + resolveArchivedThreadsForDelete, + resolveThreadTargetWithArchivedFallback, + shouldDeleteWorktreeClientSide, + ThreadArchiveBlockedError, +} from "./useThreadActions"; describe("ThreadArchiveBlockedError", () => { it("keeps the blocked thread context with the fixed message", () => { @@ -17,3 +24,110 @@ describe("ThreadArchiveBlockedError", () => { expect(error.message).toBe("Cannot archive a running thread."); }); }); + +describe("shouldDeleteWorktreeClientSide", () => { + it("keeps the legacy client-side cleanup path for older servers", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: true, + supportsDurableWorktreeCleanup: false, + }), + ).toBe(true); + }); + + it("leaves cleanup to the durable server path when supported", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: true, + supportsDurableWorktreeCleanup: true, + }), + ).toBe(false); + }); + + it("does not remove a worktree when the user keeps it", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: false, + supportsDurableWorktreeCleanup: false, + }), + ).toBe(false); + }); +}); + +describe("resolveThreadTargetWithArchivedFallback", () => { + const target = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }; + const archivedThread = { + environmentId: target.environmentId, + id: target.threadId, + worktreePath: "/tmp/archived-worktree", + }; + + it("lets archived settings deletion use the archived shell", () => { + expect(resolveThreadTargetWithArchivedFallback(target, null, [archivedThread])).toEqual({ + thread: archivedThread, + threadRef: target, + }); + }); + + it("rejects a fallback shell from another target", () => { + expect( + resolveThreadTargetWithArchivedFallback(target, null, [ + { ...archivedThread, id: ThreadId.make("other-thread") }, + ]), + ).toBeNull(); + }); +}); + +describe("collectThreadDeleteCandidates", () => { + it("keeps an archived sibling in orphan detection", () => { + const environmentId = EnvironmentId.make("environment-1"); + const target = { + environmentId, + id: ThreadId.make("thread-1"), + worktreePath: "/tmp/shared-worktree", + }; + const sibling = { + environmentId, + id: ThreadId.make("thread-2"), + worktreePath: "/tmp/shared-worktree", + }; + + const candidates = collectThreadDeleteCandidates([], target, [sibling]); + + expect(candidates).toHaveLength(2); + expect(candidates.map((thread) => thread.id)).toEqual(["thread-2", "thread-1"]); + expect(getOrphanedWorktreePathForThread(candidates, target.id)).toBeNull(); + }); +}); + +describe("resolveArchivedThreadsForDelete", () => { + it("loads archived owners for a normal worktree deletion", async () => { + const archivedThread = { + environmentId: EnvironmentId.make("environment-1"), + id: ThreadId.make("archived-owner"), + worktreePath: "/tmp/shared-worktree", + }; + + await expect( + resolveArchivedThreadsForDelete({ + worktreePath: "/tmp/shared-worktree", + load: async () => [archivedThread], + }), + ).resolves.toEqual([archivedThread]); + }); + + it("uses supplied archived shells without loading them again", async () => { + const load = () => Promise.reject(new Error("should not load")); + + await expect( + resolveArchivedThreadsForDelete({ + archivedThreads: [], + worktreePath: "/tmp/shared-worktree", + load, + }), + ).resolves.toEqual([]); + }); +}); diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index 569b4be96e62..e89adea9ad54 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -6,6 +6,7 @@ import { } from "@t3tools/client-runtime/environment"; import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { canSettle, canSnooze, threadWokeAt } from "@t3tools/client-runtime/state/thread-settled"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; import { EnvironmentId, type ScopedThreadRef, ThreadId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Schema from "effect/Schema"; @@ -19,13 +20,17 @@ import { terminalEnvironment } from "../state/terminal"; import { threadEnvironment } from "../state/threads"; import { vcsEnvironment } from "../state/vcs"; import { useNewThreadHandler } from "./useHandleNewThread"; -import { refreshArchivedThreadsForEnvironment } from "../lib/archivedThreadsState"; +import { + loadArchivedThreadsForEnvironment, + refreshArchivedThreadsForEnvironment, +} from "../lib/archivedThreadsState"; import { readLocalApi } from "../localApi"; import { readEnvironmentSupportsPinning, readEnvironmentSupportsPinReorder, readEnvironmentSupportsSettlement, readEnvironmentSupportsSnooze, + readEnvironmentSupportsWorktreeCleanup, readEnvironmentThreadRefs, readProject, readThreadShell, @@ -51,6 +56,65 @@ export class ThreadArchiveBlockedError extends Schema.TaggedErrorClass; + /** Shells supplied by archived-thread views, which are outside the active store. */ + readonly archivedThreads?: ReadonlyArray; +}; + +export function resolveThreadTargetWithArchivedFallback< + T extends Pick, +>( + target: ScopedThreadRef, + activeThread: T | null, + archivedThreads: ReadonlyArray | undefined, +): { readonly thread: T; readonly threadRef: ScopedThreadRef } | null { + const candidate = + activeThread ?? + archivedThreads?.find( + (thread) => thread.environmentId === target.environmentId && thread.id === target.threadId, + ); + if ( + candidate === undefined || + candidate.environmentId !== target.environmentId || + candidate.id !== target.threadId + ) { + return null; + } + return { thread: candidate, threadRef: target }; +} + +export function collectThreadDeleteCandidates< + T extends Pick, +>( + activeThreads: ReadonlyArray, + targetThread: T, + archivedThreads: ReadonlyArray, +): ReadonlyArray { + const candidates = new Map(); + for (const thread of [...activeThreads, ...archivedThreads, targetThread]) { + candidates.set(`${thread.environmentId}:${thread.id}`, thread); + } + return [...candidates.values()]; +} + +export function resolveArchivedThreadsForDelete(input: { + readonly archivedThreads?: ReadonlyArray; + readonly worktreePath: string | null; + readonly load: () => Promise>; +}): Promise> { + if (input.archivedThreads !== undefined) return Promise.resolve(input.archivedThreads); + if (input.worktreePath === null) return Promise.resolve([]); + return input.load(); +} + export class ThreadSettlementUnsupportedError extends Schema.TaggedErrorClass()( "ThreadSettlementUnsupportedError", { @@ -270,8 +334,12 @@ export function useThreadActions() { ); const deleteThread = useCallback( - async (target: ScopedThreadRef, opts: { deletedThreadKeys?: ReadonlySet } = {}) => { - const resolved = resolveThreadTarget(target); + async (target: ScopedThreadRef, opts: DeleteThreadOptions = {}) => { + const resolved = resolveThreadTargetWithArchivedFallback( + target, + resolveThreadTarget(target)?.thread ?? null, + opts.archivedThreads, + ); if (!resolved) { // Thread not in main store (e.g. archived thread) — dispatch delete directly. const result = await deleteThreadMutation({ @@ -284,10 +352,22 @@ export function useThreadActions() { return result; } const { thread, threadRef } = resolved; - const threads = readEnvironmentThreadRefs(threadRef.environmentId).flatMap((ref) => { + const archivedThreadsResult = await settlePromise(() => + resolveArchivedThreadsForDelete({ + ...(opts.archivedThreads === undefined ? {} : { archivedThreads: opts.archivedThreads }), + worktreePath: thread.worktreePath, + load: () => loadArchivedThreadsForEnvironment(threadRef.environmentId), + }), + ); + if (archivedThreadsResult._tag === "Failure") { + return archivedThreadsResult; + } + const archivedThreads = archivedThreadsResult.value; + const activeThreads = readEnvironmentThreadRefs(threadRef.environmentId).flatMap((ref) => { const shell = readThreadShell(ref); return shell === null ? [] : [shell]; }); + const threads = collectThreadDeleteCandidates(activeThreads, thread, archivedThreads); const threadProject = readProject({ environmentId: threadRef.environmentId, projectId: thread.projectId, @@ -312,6 +392,9 @@ export function useThreadActions() { const displayWorktreePath = orphanedWorktreePath ? formatWorktreePathForDisplay(orphanedWorktreePath) : null; + const supportsDurableWorktreeCleanup = readEnvironmentSupportsWorktreeCleanup( + threadRef.environmentId, + ); const canDeleteWorktree = orphanedWorktreePath !== null && threadProject !== null; const localApi = readLocalApi(); let shouldDeleteWorktree = false; @@ -358,7 +441,12 @@ export function useThreadActions() { }); const deleteResult = await deleteThreadMutation({ environmentId: threadRef.environmentId, - input: { threadId: threadRef.threadId }, + input: { + threadId: threadRef.threadId, + ...(shouldDeleteWorktree && supportsDurableWorktreeCleanup + ? { deleteWorktree: true } + : {}), + }, }); if (deleteResult._tag === "Failure") { return deleteResult; @@ -407,7 +495,14 @@ export function useThreadActions() { } } - if (!shouldDeleteWorktree || !orphanedWorktreePath || !threadProject) { + if ( + !shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree, + supportsDurableWorktreeCleanup, + }) || + !orphanedWorktreePath || + !threadProject + ) { return deleteResult; } @@ -450,6 +545,7 @@ export function useThreadActions() { ); return cleanupFailure; } + return deleteResult; }, [ @@ -667,9 +763,13 @@ export function useThreadActions() { ); const confirmAndDeleteThread = useCallback( - async (target: ScopedThreadRef) => { + async (target: ScopedThreadRef, opts: Pick = {}) => { const localApi = readLocalApi(); - const resolved = resolveThreadTarget(target); + const resolved = resolveThreadTargetWithArchivedFallback( + target, + resolveThreadTarget(target)?.thread ?? null, + opts.archivedThreads, + ); if (confirmThreadDelete && localApi) { const title = resolved?.thread.title ?? "this thread"; @@ -690,7 +790,7 @@ export function useThreadActions() { } } - return deleteThread(target); + return deleteThread(target, opts); }, [confirmThreadDelete, deleteThread, resolveThreadTarget], ); diff --git a/apps/web/src/lib/archivedThreadsState.ts b/apps/web/src/lib/archivedThreadsState.ts index 4087b24ac330..0f6945f44855 100644 --- a/apps/web/src/lib/archivedThreadsState.ts +++ b/apps/web/src/lib/archivedThreadsState.ts @@ -4,7 +4,10 @@ import { createArchivedThreadSnapshotsAtomFamily, makeArchivedThreadsEnvironmentKey, } from "@t3tools/client-runtime/state/threads"; -import type { EnvironmentId } from "@t3tools/contracts"; +import type { EnvironmentId, OrchestrationShellSnapshot } from "@t3tools/contracts"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; +import * as Cause from "effect/Cause"; +import { AsyncResult } from "effect/unstable/reactivity"; import { useCallback, useMemo } from "react"; import { @@ -34,6 +37,31 @@ export function refreshArchivedThreadsForEnvironment(environmentId: EnvironmentI appAtomRegistry.refresh(archivedSnapshotAtom(environmentId)); } +/** Refresh and read the archived shells before destructive ownership checks. */ +export function loadArchivedThreadsForEnvironment( + environmentId: EnvironmentId, +): Promise> { + const atom = archivedSnapshotAtom(environmentId); + appAtomRegistry.refresh(atom); + + return new Promise((resolve, reject) => { + let unsubscribe = () => {}; + const settle = (result: AsyncResult.AsyncResult) => { + if (result.waiting) return; + if (result._tag === "Success") { + unsubscribe(); + resolve(result.value.threads.map((thread) => ({ ...thread, environmentId }))); + } else if (result._tag === "Failure") { + unsubscribe(); + reject(Cause.squash(result.cause)); + } + }; + + unsubscribe = appAtomRegistry.subscribe(atom, settle); + settle(appAtomRegistry.get(atom)); + }); +} + export function useArchivedThreadSnapshots(environmentIds: ReadonlyArray): { readonly snapshots: ReadonlyArray; readonly error: string | null; diff --git a/apps/web/src/state/entities.ts b/apps/web/src/state/entities.ts index 72e37a3c6a23..6b213711d512 100644 --- a/apps/web/src/state/entities.ts +++ b/apps/web/src/state/entities.ts @@ -277,6 +277,15 @@ export function readEnvironmentSupportsThreadAnnotations(environmentId: Environm ); } +/** Whether the server durably owns worktree cleanup after thread deletion. + Missing is unsupported so an older server cannot discard the user's cleanup choice. */ +export function readEnvironmentSupportsWorktreeCleanup(environmentId: EnvironmentId): boolean { + return ( + appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities + .threadWorktreeCleanup === true + ); +} + export function readThreadDetail(ref: ScopedThreadRef): EnvironmentThread | null { return appAtomRegistry.get(environmentThreadDetails.detailAtom(ref)); } diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index c90ef9db25c0..243f86bd4b2a 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -29,6 +29,18 @@ changed. Editing, resolving, or reopening the annotation moves that marker to th Resolved annotations disappear from the sidebar and composer but remain available from their yellow minimap marker, where they can be edited or reopened. +## Worktree cleanup in LastCode + +When you delete a thread and choose to delete its worktree, the thread stays in the sidebar until +the server finishes that cleanup. **Deleting** means removal is active. **Deleting (Queued)** means +another worktree from the same repository is being removed first; hover the row to see which +thread it is waiting for. Cleanup for different repositories can proceed at the same time. + +If cleanup fails, the row changes to **Cleanup failed**. Select anywhere on that row to see the +error and choose **Retry**, **Copy details**, or **Keep worktree**. LastCode resumes unfinished +cleanup after a server restart. On mobile, long-press a failed row to choose **Retry** or +**Keep worktree**. + ## Environment artwork Dev and Nightly environments can identify themselves with artwork at the top of the sidebar and in diff --git a/packages/client-runtime/src/operations/commands.ts b/packages/client-runtime/src/operations/commands.ts index bbbd3b873fa5..2b6f5bd8f939 100644 --- a/packages/client-runtime/src/operations/commands.ts +++ b/packages/client-runtime/src/operations/commands.ts @@ -33,6 +33,8 @@ export type UpdateProjectInput = CommandInput<"project.meta.update">; export type DeleteProjectInput = CommandInput<"project.delete">; export type CreateThreadInput = CommandInput<"thread.create">; export type DeleteThreadInput = CommandInput<"thread.delete">; +export type RetryThreadWorktreeCleanupInput = CommandInput<"thread.worktree-cleanup.retry">; +export type AbandonThreadWorktreeCleanupInput = CommandInput<"thread.worktree-cleanup.abandon">; export type ArchiveThreadInput = CommandInput<"thread.archive">; export type UnarchiveThreadInput = CommandInput<"thread.unarchive">; export type SettleThreadInput = CommandInput<"thread.settle">; @@ -143,6 +145,27 @@ export const deleteThread: (input: DeleteThreadInput) => CommandEffect = Effect. }); }); +export const retryThreadWorktreeCleanup: (input: RetryThreadWorktreeCleanupInput) => CommandEffect = + Effect.fn("EnvironmentCommands.retryThreadWorktreeCleanup")(function* (input) { + return yield* dispatch({ + ...input, + type: "thread.worktree-cleanup.retry", + commandId: yield* commandId(input), + }); + }); + +export const abandonThreadWorktreeCleanup: ( + input: AbandonThreadWorktreeCleanupInput, +) => CommandEffect = Effect.fn("EnvironmentCommands.abandonThreadWorktreeCleanup")( + function* (input) { + return yield* dispatch({ + ...input, + type: "thread.worktree-cleanup.abandon", + commandId: yield* commandId(input), + }); + }, +); + export const archiveThread: (input: ArchiveThreadInput) => CommandEffect = Effect.fn( "EnvironmentCommands.archiveThread", )(function* (input) { diff --git a/packages/client-runtime/src/state/threadCommands.ts b/packages/client-runtime/src/state/threadCommands.ts index 6346f863158b..18f23ce80429 100644 --- a/packages/client-runtime/src/state/threadCommands.ts +++ b/packages/client-runtime/src/state/threadCommands.ts @@ -11,6 +11,8 @@ import { type ArchiveThreadInput, type CreateThreadInput, type DeleteThreadInput, + type RetryThreadWorktreeCleanupInput, + type AbandonThreadWorktreeCleanupInput, type InterruptThreadTurnInput, type RespondToThreadApprovalInput, type RespondToThreadUserInputInput, @@ -34,6 +36,8 @@ import { archiveThread, createThread, deleteThread, + retryThreadWorktreeCleanup, + abandonThreadWorktreeCleanup, interruptThreadTurn, respondToThreadApproval, respondToThreadUserInput, @@ -61,6 +65,8 @@ export type { ArchiveThreadInput, CreateThreadInput, DeleteThreadInput, + RetryThreadWorktreeCleanupInput, + AbandonThreadWorktreeCleanupInput, InterruptThreadTurnInput, RespondToThreadApprovalInput, RespondToThreadUserInputInput, @@ -105,6 +111,18 @@ export function createThreadEnvironmentAtoms( scheduler, concurrency, }), + retryWorktreeCleanup: createEnvironmentCommand(runtime, { + label: "environment-data:commands:thread:worktree-cleanup:retry", + execute: (input: RetryThreadWorktreeCleanupInput) => retryThreadWorktreeCleanup(input), + scheduler, + concurrency, + }), + abandonWorktreeCleanup: createEnvironmentCommand(runtime, { + label: "environment-data:commands:thread:worktree-cleanup:abandon", + execute: (input: AbandonThreadWorktreeCleanupInput) => abandonThreadWorktreeCleanup(input), + scheduler, + concurrency, + }), archive: createEnvironmentCommand(runtime, { label: "environment-data:commands:thread:archive", execute: (input: ArchiveThreadInput) => archiveThread(input), diff --git a/packages/contracts/src/environment.test.ts b/packages/contracts/src/environment.test.ts index 35f94443f35d..3f745620a983 100644 --- a/packages/contracts/src/environment.test.ts +++ b/packages/contracts/src/environment.test.ts @@ -36,4 +36,14 @@ describe("ExecutionEnvironmentDescriptor", () => { }).capabilities.threadAnnotations, ).toBe(true); }); + + it("treats missing worktree cleanup as unsupported and preserves support", () => { + expect(decodeDescriptor(descriptor).capabilities.threadWorktreeCleanup).toBeUndefined(); + expect( + decodeDescriptor({ + ...descriptor, + capabilities: { ...descriptor.capabilities, threadWorktreeCleanup: true }, + }).capabilities.threadWorktreeCleanup, + ).toBe(true); + }); }); diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 2246d71a78ef..de424f6c89b5 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -70,6 +70,9 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ /** Server understands thread annotation create/edit/resolve/reopen commands and exposes annotation state in thread shell/detail snapshots. */ threadAnnotations: Schema.optionalKey(Schema.Boolean), + /** Server durably owns thread worktree cleanup after deletion and understands + deleteWorktree plus cleanup retry/abandon commands. */ + threadWorktreeCleanup: Schema.optionalKey(Schema.Boolean), /** The update path clients should offer for this server. Absent on servers that must be relaunched manually (dev checkouts, Windows foreground runs, pre-update servers). */ diff --git a/packages/contracts/src/git.ts b/packages/contracts/src/git.ts index 915c3627c9b9..011d3453c3c8 100644 --- a/packages/contracts/src/git.ts +++ b/packages/contracts/src/git.ts @@ -161,6 +161,7 @@ export const VcsRemoveWorktreeInput = Schema.Struct({ cwd: TrimmedNonEmptyStringSchema, path: TrimmedNonEmptyStringSchema, force: Schema.optional(Schema.Boolean), + allowMissing: Schema.optional(Schema.Boolean), }); export type VcsRemoveWorktreeInput = typeof VcsRemoveWorktreeInput.Type; diff --git a/packages/contracts/src/orchestration.test.ts b/packages/contracts/src/orchestration.test.ts index c185fb7b3ee6..95ae07fac97e 100644 --- a/packages/contracts/src/orchestration.test.ts +++ b/packages/contracts/src/orchestration.test.ts @@ -72,6 +72,47 @@ it.effect("decodes a dispatch error after its bootstrap thread was deleted", () }), ); +it.effect("decodes durable worktree cleanup commands and events", () => + Effect.gen(function* () { + const command = yield* decodeOrchestrationCommand({ + type: "thread.delete", + commandId: "cmd-delete-worktree", + threadId: "thread-cleanup", + deleteWorktree: true, + }); + assert.strictEqual(command.type, "thread.delete"); + if (command.type === "thread.delete") assert.strictEqual(command.deleteWorktree, true); + + const event = yield* decodeOrchestrationEvent({ + sequence: 1, + eventId: "event-delete-worktree", + aggregateKind: "thread", + aggregateId: "thread-cleanup", + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: "cmd-delete-worktree", + causationEventId: null, + correlationId: "cmd-delete-worktree", + metadata: {}, + payload: { + threadId: "thread-cleanup", + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: "2026-08-23T00:00:00.000Z", + blockedByThreadId: "thread-blocker", + }, + }, + }); + assert.strictEqual(event.type, "thread.deleted"); + if (event.type === "thread.deleted") { + assert.strictEqual(event.payload.worktreeCleanup?.status, "queued"); + } + }), +); + it.effect("parses turn diff input when fromTurnCount <= toTurnCount", () => Effect.gen(function* () { const parsed = yield* decodeTurnDiffInput({ diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 0f478707db5f..d99651cf1791 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -462,6 +462,39 @@ export const ThreadAnnotation = Schema.Struct({ }); export type ThreadAnnotation = typeof ThreadAnnotation.Type; +const ThreadWorktreeCleanupBase = { + repositoryRoot: TrimmedNonEmptyString, + /** + * Canonical Git common directory used to serialize worktree operations. + * Optional for cleanup rows written before repository identity was added; + * those rows fall back to repositoryRoot when selecting a blocker/worker. + */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), + worktreePath: TrimmedNonEmptyString, +} as const; + +export const ThreadWorktreeCleanup = Schema.Union([ + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("deleting"), + startedAt: IsoDateTime, + }), + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("queued"), + queuedAt: IsoDateTime, + blockedByThreadId: ThreadId, + }), + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("failed"), + startedAt: IsoDateTime, + failedAt: IsoDateTime, + error: Schema.String, + }), +]); +export type ThreadWorktreeCleanup = typeof ThreadWorktreeCleanup.Type; + export const OrchestrationThread = Schema.Struct({ id: ThreadId, projectId: ProjectId, @@ -504,6 +537,7 @@ export const OrchestrationThread = Schema.Struct({ // Command decisions use this projected marker to anchor annotations without // hydrating message bodies and attachments for every thread. latestUserMessageId: Schema.optional(Schema.NullOr(MessageId)), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), deletedAt: Schema.NullOr(IsoDateTime), messages: Schema.Array(OrchestrationMessage), proposedPlans: Schema.Array(OrchestrationProposedPlan).pipe( @@ -563,6 +597,7 @@ export const OrchestrationThreadShell = Schema.Struct({ pinOrderKey: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), titleRegeneration: Schema.optional(Schema.NullOr(ThreadTitleRegeneration)), annotation: Schema.optional(Schema.NullOr(ThreadAnnotation)), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), session: Schema.NullOr(OrchestrationSession), latestUserMessageAt: Schema.NullOr(IsoDateTime), hasPendingApprovals: Schema.Boolean, @@ -762,6 +797,8 @@ const ProjectDeleteCommand = Schema.Struct({ commandId: CommandId, projectId: ProjectId, force: Schema.optional(Schema.Boolean), + /** Resolved by command normalization for forced worktree cleanup. */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), }); const ThreadCreateCommand = Schema.Struct({ @@ -784,6 +821,21 @@ const ThreadDeleteCommand = Schema.Struct({ type: Schema.Literal("thread.delete"), commandId: CommandId, threadId: ThreadId, + deleteWorktree: Schema.optional(Schema.Boolean), + /** Resolved by command normalization from the thread's project checkout. */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), +}); + +const ThreadWorktreeCleanupRetryCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.retry"), + commandId: CommandId, + threadId: ThreadId, +}); + +const ThreadWorktreeCleanupAbandonCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.abandon"), + commandId: CommandId, + threadId: ThreadId, }); const ThreadArchiveCommand = Schema.Struct({ @@ -1038,6 +1090,8 @@ const DispatchableClientOrchestrationCommand = Schema.Union([ ProjectDeleteCommand, ThreadCreateCommand, ThreadDeleteCommand, + ThreadWorktreeCleanupRetryCommand, + ThreadWorktreeCleanupAbandonCommand, ThreadArchiveCommand, ThreadUnarchiveCommand, ThreadSettleCommand, @@ -1069,6 +1123,8 @@ export const ClientOrchestrationCommand = Schema.Union([ ProjectDeleteCommand, ThreadCreateCommand, ThreadDeleteCommand, + ThreadWorktreeCleanupRetryCommand, + ThreadWorktreeCleanupAbandonCommand, ThreadArchiveCommand, ThreadUnarchiveCommand, ThreadSettleCommand, @@ -1193,6 +1249,13 @@ const ThreadTurnAssistantFinalizeCommand = Schema.Struct({ createdAt: IsoDateTime, }); +const ThreadWorktreeCleanupUpdateCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.update"), + commandId: CommandId, + threadId: ThreadId, + cleanup: Schema.NullOr(ThreadWorktreeCleanup), +}); + const InternalOrchestrationCommand = Schema.Union([ ThreadSessionSetCommand, ThreadMessageAssistantDeltaCommand, @@ -1204,6 +1267,7 @@ const InternalOrchestrationCommand = Schema.Union([ ThreadTitleRegenerationCompleteCommand, ThreadTurnRequestResolveCommand, ThreadTurnAssistantFinalizeCommand, + ThreadWorktreeCleanupUpdateCommand, ]); export type InternalOrchestrationCommand = typeof InternalOrchestrationCommand.Type; @@ -1219,6 +1283,7 @@ export const OrchestrationEventType = Schema.Literals([ "project.deleted", "thread.created", "thread.deleted", + "thread.worktree-cleanup-updated", "thread.archived", "thread.unarchived", "thread.settled", @@ -1303,6 +1368,13 @@ export const ThreadCreatedPayload = Schema.Struct({ export const ThreadDeletedPayload = Schema.Struct({ threadId: ThreadId, deletedAt: IsoDateTime, + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), +}); + +export const ThreadWorktreeCleanupUpdatedPayload = Schema.Struct({ + threadId: ThreadId, + cleanup: Schema.NullOr(ThreadWorktreeCleanup), + updatedAt: IsoDateTime, }); export const ThreadArchivedPayload = Schema.Struct({ @@ -1560,6 +1632,11 @@ export const OrchestrationEvent = Schema.Union([ type: Schema.Literal("thread.deleted"), payload: ThreadDeletedPayload, }), + Schema.Struct({ + ...EventBaseFields, + type: Schema.Literal("thread.worktree-cleanup-updated"), + payload: ThreadWorktreeCleanupUpdatedPayload, + }), Schema.Struct({ ...EventBaseFields, type: Schema.Literal("thread.archived"), diff --git a/packages/shared/src/DrainableWorker.test.ts b/packages/shared/src/DrainableWorker.test.ts index 8e4c654e2e4c..f87e859dbf45 100644 --- a/packages/shared/src/DrainableWorker.test.ts +++ b/packages/shared/src/DrainableWorker.test.ts @@ -54,4 +54,27 @@ describe("makeDrainableWorker", () => { }), ), ); + + it.live("shuts down the worker and queue without retaining later work", () => + Effect.scoped( + Effect.gen(function* () { + const processed: string[] = []; + const worker = yield* makeDrainableWorker((item: string) => + Effect.sync(() => processed.push(item)), + ); + + yield* worker.enqueue("before-shutdown"); + yield* worker.drain; + yield* worker.shutdown; + + // Offers after retirement are rejected by the closed queue and must + // not make drain wait forever. Closing the parent scope repeats the + // queue finalizer, proving shutdown is safe to call twice. + yield* worker.enqueue("after-shutdown"); + yield* worker.drain; + + expect(processed).toEqual(["before-shutdown"]); + }), + ), + ); }); diff --git a/packages/shared/src/DrainableWorker.ts b/packages/shared/src/DrainableWorker.ts index de40ec5e36b8..d126c1e9f335 100644 --- a/packages/shared/src/DrainableWorker.ts +++ b/packages/shared/src/DrainableWorker.ts @@ -10,6 +10,7 @@ */ import * as Scope from "effect/Scope"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as TxQueue from "effect/TxQueue"; import * as TxRef from "effect/TxRef"; @@ -26,6 +27,14 @@ export interface DrainableWorker { * Resolves when the queue is empty and the worker is idle (not processing). */ readonly drain: Effect.Effect; + + /** + * Stop the worker after its current queue has been drained. + * + * Callers that coordinate access to a worker may use this to retire idle + * keyed workers before their parent scope closes. + */ + readonly shutdown: Effect.Effect; } /** @@ -41,7 +50,10 @@ export const makeDrainableWorker = ( process: (item: A) => Effect.Effect, ): Effect.Effect, never, Scope.Scope | R> => Effect.gen(function* () { - const queue = yield* Effect.acquireRelease(TxQueue.unbounded(), TxQueue.shutdown); + const workerScope = yield* Scope.make("sequential"); + yield* Effect.addFinalizer(() => Scope.close(workerScope, Exit.void).pipe(Effect.ignore)); + const queue = yield* TxQueue.unbounded(); + yield* Scope.addFinalizer(workerScope, TxQueue.shutdown(queue).pipe(Effect.asVoid)); const outstanding = yield* TxRef.make(0); yield* TxQueue.take(queue).pipe( @@ -52,7 +64,7 @@ export const makeDrainableWorker = ( ), ), Effect.forever, - Effect.forkScoped, + Effect.forkIn(workerScope), ); const drain: DrainableWorker["drain"] = TxRef.get(outstanding).pipe( @@ -60,11 +72,18 @@ export const makeDrainableWorker = ( Effect.tx, ); - const enqueue = (element: A): Effect.Effect => + const enqueue = (element: A): Effect.Effect => TxQueue.offer(queue, element).pipe( - Effect.tap(() => TxRef.update(outstanding, (n) => n + 1)), + Effect.tap((accepted) => + accepted ? TxRef.update(outstanding, (n) => n + 1) : Effect.void, + ), + Effect.asVoid, Effect.tx, ); - return { enqueue, drain } satisfies DrainableWorker; + // Closing the child scope interrupts the worker and shuts down the queue. + // The parent scope also closes it, and Scope.close is idempotent. + const shutdown = Scope.close(workerScope, Exit.void).pipe(Effect.ignore); + + return { enqueue, drain, shutdown } satisfies DrainableWorker; });