From d10f888462b56f1ccaa455326e68bed3d89bce5f Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Sun, 23 Aug 2026 16:30:05 -0700 Subject: [PATCH 01/22] docs(lastcode): plan durable worktree cleanup --- .../lastcode/durable-worktree-cleanup-plan.md | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 docs/lastcode/durable-worktree-cleanup-plan.md diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md new file mode 100644 index 000000000000..cd64c48531a4 --- /dev/null +++ b/docs/lastcode/durable-worktree-cleanup-plan.md @@ -0,0 +1,128 @@ +# Durable Thread Worktree Cleanup Plan + +## Goal + +Deleting a thread with “Delete the worktree too” must create a durable cleanup job in the same persisted domain event as the thread deletion. The server owns that job until it succeeds or the user explicitly chooses **Keep worktree**. Closing a client, losing the connection, or restarting the server must not lose the cleanup. + +The deleted thread remains visible as a temporary tombstone while cleanup is deleting, queued, or failed. Successful cleanup and explicit abandonment remove the tombstone silently. + +## Product behavior + +### Confirmation and navigation + +- Keep the existing two-step delete confirmation. +- Web and desktop may request worktree cleanup. Mobile may delete a thread but does not offer worktree deletion. +- The server derives the repository root and worktree path from the authoritative project and thread records. The client sends only `deleteWorktree: true`; it does not supply paths. +- Reject a cleanup request unless the thread owns a linked worktree that no other live thread uses. +- Once deletion is accepted, immediately navigate an active thread route to the existing fallback. The tombstone is a sidebar status item, not an openable chat. +- Archive never schedules worktree cleanup. + +### Lifecycle and queueing + +- Persist cleanup intent and its initial state atomically in `thread.deleted`. +- Cleanup states are a separate lifecycle axis from agent states such as Working, Waiting, approvals, and input requests. +- The visible states are: + - `deleting`: orange **Deleting**; the v2 sidebar also shows elapsed time. + - `queued`: orange **Deleting (Queued)** with the blocking thread ID and title in hover details. + - `failed`: red **Cleanup failed**; any click on the row opens the failure dialog. +- Cleanups for the same repository run one at a time in deletion order. Cleanups for unrelated repositories may run concurrently. +- On server start, reload deleting and queued jobs from the projection and resume them in deletion order. Treat a stale deleting state as resumable work. +- A failed cleanup releases its repository queue so the next queued cleanup can run. +- Retry re-enters the repository queue. If another cleanup owns it, persist queued state and its blocker; otherwise persist deleting state and start immediately. +- Success clears the cleanup state and removes the tombstone without a toast. +- **Keep worktree** is the only abandonment path. It clears the cleanup state and removes the tombstone without deleting the worktree. + +### Failure dialog + +- Clicking anywhere on a failed row opens a dialog containing the thread identity, worktree path, and exact cleanup error. +- Actions are **Retry**, **Copy details**, and destructive-looking but non-destructive **Keep worktree**. +- Retry closes the dialog after the server accepts it. Copy details keeps it open. Keep worktree requires confirmation because it permanently stops automatic cleanup. + +## Domain and persistence design + +### Contracts + +- Extend `thread.delete` with optional `deleteWorktree: boolean`. +- Add a `ThreadWorktreeCleanup` discriminated union to thread shell/detail contracts. Each variant carries the authoritative repository root and worktree path: + - deleting: `startedAt` + - queued: `queuedAt`, `blockedByThreadId` + - failed: `startedAt`, `failedAt`, `error` +- Add client commands for retry and abandonment, plus internal commands/events for queued, started, failed, and completed transitions. +- Keep new fields optional on the wire where compatibility with cached snapshots or older clients is required; absent means no cleanup. + +### Decision rules + +- `thread.delete` derives and validates cleanup ownership from the command read model. +- The decider chooses the initial deleting or queued state by inspecting unfinished cleanup jobs for the same repository. The resulting `thread.deleted` event contains the concrete cleanup record, making the user’s choice durable with the deletion. +- Retry is valid only from failed. Abandonment is valid from deleting, queued, or failed. Internal lifecycle transitions validate their expected prior state. +- A failed job is not an active queue blocker. + +### Projection + +- Add a nullable JSON cleanup column to `projection_threads` through the next migration. +- Project every cleanup transition into that column. +- Include soft-deleted rows in shell snapshots and per-thread shell lookups only while cleanup is non-null. Completed or abandoned jobs disappear through the existing `thread-removed` shell event. +- Make `thread.deleted` use the same projection-backed upsert-or-remove decision as other thread events so the initial tombstone reaches every connected client. +- Expose a repository query for resumable deleting/queued jobs ordered by deletion time and thread ID. + +### Reactor + +- Extend `ThreadDeletionReactor`; it already owns provider-session and terminal cleanup for `thread.deleted`. +- Queue one scoped cleanup fiber per job behind a repository-keyed semaphore. Semaphore acquisition preserves same-repository serialization while different repositories proceed independently. +- Before a queued fiber runs, persist the started transition. Call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. +- Persist completed or failed with the exact structured error message. Refresh Git status after success as best effort; refresh failure must not turn a successful deletion into a failed cleanup. +- At startup, query and enqueue all resumable jobs after projections are bootstrapped. Tests wait on the reactor drain/receipts, never sleeps. + +## Client presentation + +### Shared behavior + +- Cleanup state overrides agent status once deletion is accepted. +- Deleting and queued rows are muted, non-selectable, and use a no-entry cursor. Failed rows are clickable only to open their dialog. +- Preserve the thread title, short ID, project grouping, branch, and muted `FolderGit2Icon`. Do not invent a new worktree glyph or color the worktree icon orange. +- Exclude tombstones from keyboard thread traversal, bulk thread actions, drag/reorder, route fallback candidates, and unread/settled calculations. + +### Legacy sidebar + +- Render orange `• Deleting` and `• Deleting (Queued)` labels with the existing status-label proportions. +- Render red `• Cleanup failed`; the whole row is the dialog target. +- Append a cleanup segment after the regular hover content and after any annotation: + - deleting: `Deleting worktree` and the formatted path. + - queued: `Waiting for — `. +- The segment background uses the deleting orange in light and dark themes. Text uses normal foreground in light themes and the darkest normal background token in dark themes. + +### V2 sidebar + +- Reuse the top-right status slot and dashed-circle visual language. +- Deleting shows orange `Deleting <elapsed>`. +- Queued shows orange `Deleting (Queued)` with no timer. +- Failed shows red `Cleanup failed`. +- Use the same shared hover content as the legacy sidebar. + +### Mobile + +- Decode and retain cleanup tombstones received from the server. +- Show deleting, queued, and failed cleanup status ahead of normal thread status resolution. +- Do not add a mobile control that initiates worktree deletion. Server restart/reconnect recovery remains fully visible on mobile. + +## Validation + +- Contract encode/decode tests for every cleanup state and command/event. +- Decider tests for ownership, shared-worktree rejection, atomic initial state, retry, abandonment, and transition invariants. +- Projection/migration/query tests for persistence, startup enumeration, shell inclusion, and final removal. +- Reactor tests with controlled removal effects proving same-repository serialization, cross-repository concurrency, restart recovery, success, failure, retry, and drain semantics. +- Client-runtime reducer tests proving tombstones survive upserts and disappear on completion/abandonment. +- Web logic/component tests for status priority, disabled/clickable semantics, hover order/content, failure dialog actions, and both sidebar variants. +- Mobile presentation tests proving observe-only status priority. +- Focused lint and package typechecks, committed-range `git diff --check`, and the guarded LastCode quick-CI push gate. +- Integrated disposable-state web QA in both sidebar versions, light and dark themes, covering deleting, queued, failure, Retry, Copy details, Keep worktree, navigation fallback, and silent success. Capture before/after images; record motion if elapsed/transition behavior needs review. + +## Surface matrix + +- Web: initiates cleanup and renders/controls the full lifecycle. +- Desktop: inherits web behavior; no new Electron IPC. +- Mobile: observes lifecycle but cannot initiate worktree deletion. +- Providers: not provider-shaped; existing deletion reactor stops any provider session first. +- Contracts/server/projections: required for durable remote and multi-device behavior. +- Local, relay, and tunnel connections: use the same persisted orchestration commands and shell stream; disconnects do not own job lifetime. +- Documentation: this LastCode implementation plan records fork-only behavior; no upstream user documentation changes in this PR. From f6c19b49325d02ac0dec59e4f5c1648416e07800 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Sun, 23 Aug 2026 18:12:47 -0700 Subject: [PATCH 02/22] feat(lastcode): make worktree cleanup durable --- .../features/threads/thread-list-items.tsx | 9 +- .../features/threads/thread-list-v2-items.tsx | 28 ++- .../src/features/threads/threadListV2.test.ts | 31 +++ .../features/threads/threadPresentation.ts | 39 ++++ .../Layers/ProjectionPipeline.ts | 17 ++ .../Layers/ProjectionSnapshotQuery.test.ts | 46 +++- .../Layers/ProjectionSnapshotQuery.ts | 25 ++- .../Layers/ThreadDeletionReactor.test.ts | 173 ++++++++++++++- .../Layers/ThreadDeletionReactor.ts | 191 +++++++++++++++- apps/server/src/orchestration/Schemas.ts | 3 + .../src/orchestration/decider.delete.test.ts | 198 +++++++++++++++++ apps/server/src/orchestration/decider.ts | 207 +++++++++++++++++- .../src/orchestration/projector.test.ts | 1 + apps/server/src/orchestration/projector.ts | 19 ++ .../persistence/Layers/ProjectionThreads.ts | 59 ++++- apps/server/src/persistence/Migrations.ts | 2 + ...46_ProjectionThreadWorktreeCleanup.test.ts | 55 +++++ .../046_ProjectionThreadWorktreeCleanup.ts | 16 ++ .../persistence/Services/ProjectionThreads.ts | 8 + apps/server/src/ws.ts | 3 +- apps/web/src/components/LegacySidebar.tsx | 73 +++++- apps/web/src/components/Sidebar.logic.test.ts | 95 ++++++++ apps/web/src/components/Sidebar.logic.ts | 53 ++++- apps/web/src/components/Sidebar.tsx | 174 ++++++++++----- .../WorktreeCleanupFailureDialog.tsx | 97 ++++++++ .../sidebar/SidebarThreadHoverContent.tsx | 37 ++++ .../thread-annotation/ThreadAnnotation.tsx | 2 + apps/web/src/hooks/useThreadActions.ts | 60 +---- .../lastcode/durable-worktree-cleanup-plan.md | 15 +- docs/user/thread-sidebar.md | 12 + .../client-runtime/src/operations/commands.ts | 23 ++ .../src/state/threadCommands.ts | 18 ++ packages/contracts/src/orchestration.test.ts | 41 ++++ packages/contracts/src/orchestration.ts | 67 ++++++ 34 files changed, 1724 insertions(+), 173 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts create mode 100644 apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts create mode 100644 apps/web/src/components/WorktreeCleanupFailureDialog.tsx diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index 2097bd4ef79a..00b394afad8d 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -459,6 +459,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { const { thread, onSelectThread, onArchiveThread, onDeleteThread, onRegenerateThreadTitle } = props; + const cleanupPending = thread.worktreeCleanup != null; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); const status = resolveThreadStatus(thread); @@ -598,7 +599,9 @@ export const ThreadListRow = memo(function ThreadListRow(props: { accessibilityHint="Swipe left for archive and delete actions" accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" + accessibilityState={{ disabled: cleanupPending }} className="bg-screen" + disabled={cleanupPending} onPress={() => { close(); onSelectThread(thread); @@ -651,7 +654,8 @@ export const ThreadListRow = memo(function ThreadListRow(props: { accessibilityHint="Opens the thread" accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" - accessibilityState={{ selected }} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} onHoverIn={() => setHovered(true)} onHoverOut={() => setHovered(false)} onPress={() => { @@ -711,6 +715,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { return ( <ThreadSwipeable backgroundColor={backgroundColor} + enabled={!cleanupPending} containerStyle={ compact ? undefined : { borderRadius: SIDEBAR_ROW_RADIUS, overflow: "hidden" } } @@ -734,7 +739,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { // ControlPillMenu injects onLongPress into the row and anchors the // token-styled dropdown to it; taps and swipes are untouched. <ControlPillMenu - actions={menuActions} + actions={cleanupPending ? [] : menuActions} onPressAction={handleMenuAction} shouldOpenOnLongPress > diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 664c8d9287e7..074b53e10f90 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -406,6 +406,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const snoozedRow = props.snoozed === true; const pinnedRow = props.pinned === true; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; + const cleanupPending = thread.worktreeCleanup != null; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); const pr = useThreadPr(thread, props.projectCwd ?? props.project?.workspaceRoot ?? null); @@ -848,7 +849,8 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { accessibilityHint={swipeAccessibilityHint} accessibilityLabel={thread.title} accessibilityRole="button" - accessibilityState={{ selected }} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} onPress={() => { close(); onSelectThread(thread); @@ -888,7 +890,8 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { accessibilityHint={swipeAccessibilityHint} accessibilityLabel={thread.title} accessibilityRole="button" - accessibilityState={{ selected }} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} className={sidebarPane ? undefined : "bg-screen"} onPress={() => { close(); @@ -971,6 +974,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { sidebarPane ? { borderRadius: SIDEBAR_V2_ROW_RADIUS, overflow: "hidden" } : undefined } enableTrackpadSwipe + enabled={!cleanupPending} // Full swipe commits the advertised lifecycle action (Settle / // Un-settle), never the secondary snooze action. fullSwipeAction="primary" @@ -987,15 +991,17 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { {(close) => ( <ControlPillMenu actions={ - snoozedRow - ? snoozedMenuActions - : !props.settlementSupported - ? legacyMenuActions - : canUnsettle - ? slimMenuActions - : swipeActions.secondary === "snooze" - ? snoozableCardMenuActions - : cardMenuActions + cleanupPending + ? [] + : snoozedRow + ? snoozedMenuActions + : !props.settlementSupported + ? legacyMenuActions + : canUnsettle + ? slimMenuActions + : swipeActions.secondary === "snooze" + ? snoozableCardMenuActions + : cardMenuActions } onPressAction={handleMenuAction} shouldOpenOnLongPress diff --git a/apps/mobile/src/features/threads/threadListV2.test.ts b/apps/mobile/src/features/threads/threadListV2.test.ts index 9f3135c50672..c72cc159d6c9 100644 --- a/apps/mobile/src/features/threads/threadListV2.test.ts +++ b/apps/mobile/src/features/threads/threadListV2.test.ts @@ -127,6 +127,37 @@ describe("resolveThreadListV2Enabled", () => { }); describe("resolveThreadListV2Status", () => { + it("shows durable cleanup before agent status", () => { + const deleting = makeThread({ + id: ThreadId.make("cleanup"), + title: "Cleanup", + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + }, + }); + expect(resolveThreadStatus(deleting)).toMatchObject({ + kind: "cleanup-deleting", + label: "Deleting", + pulse: false, + }); + expect( + resolveThreadStatus({ + ...deleting, + worktreeCleanup: { + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }, + }), + ).toMatchObject({ kind: "cleanup-failed", label: "Cleanup failed" }); + }); + it("prioritizes approval over a running session", () => { const thread = makeThread({ id: ThreadId.make("t"), diff --git a/apps/mobile/src/features/threads/threadPresentation.ts b/apps/mobile/src/features/threads/threadPresentation.ts index c556dc65e3ac..52bab7f93cb2 100644 --- a/apps/mobile/src/features/threads/threadPresentation.ts +++ b/apps/mobile/src/features/threads/threadPresentation.ts @@ -14,6 +14,9 @@ export type ThreadStatusKind = | "waiting" | "connecting" | "error" + | "cleanup-deleting" + | "cleanup-queued" + | "cleanup-failed" | "plan-ready"; export interface ThreadStatusPresentation extends StatusTone { @@ -50,6 +53,42 @@ function isLatestTurnSettled( export function resolveThreadStatus( thread: EnvironmentThreadShell, ): ThreadStatusPresentation | null { + if (thread.worktreeCleanup?.status === "failed") { + return { + kind: "cleanup-failed", + label: "Cleanup failed", + pillClassName: "bg-rose-500/12 dark:bg-rose-500/16", + textClassName: "text-rose-700 dark:text-rose-300", + iconColor: "#ff453a", + iconBackground: "rgba(255,69,58,0.22)", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "queued") { + return { + kind: "cleanup-queued", + label: "Deleting (Queued)", + pillClassName: "bg-orange-500/12 dark:bg-orange-500/16", + textClassName: "text-orange-700 dark:text-orange-300", + iconColor: "#ff9f0a", + iconBackground: "rgba(255,159,10,0.22)", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "deleting") { + return { + kind: "cleanup-deleting", + label: "Deleting", + pillClassName: "bg-orange-500/12 dark:bg-orange-500/16", + textClassName: "text-orange-700 dark:text-orange-300", + iconColor: "#ff9f0a", + iconBackground: "rgba(255,159,10,0.22)", + pulse: false, + }; + } + if (thread.hasPendingApprovals) { return { kind: "pending-approval", diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 5575bb40c20f..d350e85e4f66 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -635,6 +635,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti titleRegenerationRequestId: null, titleRegenerationStartedAt: null, annotation: null, + worktreeCleanup: null, latestUserMessageId: null, latestUserMessageAt: null, pendingApprovalCount: 0, @@ -874,11 +875,27 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti yield* projectionThreadRepository.upsert({ ...existingRow.value, deletedAt: event.payload.deletedAt, + worktreeCleanup: event.payload.worktreeCleanup ?? null, updatedAt: event.payload.deletedAt, }); return; } + case "thread.worktree-cleanup-updated": { + const existingRow = yield* projectionThreadRepository.getById({ + threadId: event.payload.threadId, + }); + if (Option.isNone(existingRow)) { + return; + } + yield* projectionThreadRepository.upsert({ + ...existingRow.value, + worktreeCleanup: event.payload.cleanup, + updatedAt: event.payload.updatedAt, + }); + return; + } + case "thread.message-sent": case "thread.proposed-plan-upserted": case "thread.activity-appended": diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 9cad0607b1b8..6ff5d6ea24ec 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -1630,6 +1630,28 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { const shellSnapshot = yield* snapshotQuery.getShellSnapshot(); assert.equal(shellSnapshot.projects.length, 0); assert.equal(shellSnapshot.threads.length, 0); + + yield* sql` + UPDATE projection_projects + SET deleted_at = NULL + WHERE project_id = 'project-deleted' + `; + yield* sql` + UPDATE projection_threads + SET + worktree_path = '/tmp/deleted-project-worktrees/thread-deleted', + worktree_cleanup_json = '{"status":"deleting","repositoryRoot":"/tmp/deleted-project","worktreePath":"/tmp/deleted-project-worktrees/thread-deleted","startedAt":"2026-04-05T00:00:05.000Z"}' + WHERE thread_id = 'thread-deleted' + `; + + const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); + assert.equal(cleanupShellSnapshot.projects.length, 1); + assert.deepStrictEqual(cleanupShellSnapshot.threads[0]?.worktreeCleanup, { + status: "deleting", + repositoryRoot: "/tmp/deleted-project", + worktreePath: "/tmp/deleted-project-worktrees/thread-deleted", + startedAt: "2026-04-05T00:00:05.000Z", + }); }), ); @@ -2444,12 +2466,14 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) = const detailWithPinnedRequests = yield* snapshotQuery.getThreadDetailById(threadW); assert.equal(detailWithPinnedRequests._tag, "Some"); if (detailWithPinnedRequests._tag === "Some") { - const ids = detailWithPinnedRequests.value.activities.map((activity) => activity.id); + const ids = new Set( + detailWithPinnedRequests.value.activities.map((activity) => activity.id), + ); assert.equal(detailWithPinnedRequests.value.activities.length, 503); - assert.equal(ids.includes(asEventId("approval-old")), true); - assert.equal(ids.includes(asEventId("user-input-old")), true); - assert.equal(ids.includes(asEventId("user-input-closed")), false); - assert.equal(ids.includes(asEventId("user-input-tied-z-request")), true); + assert.equal(ids.has(asEventId("approval-old")), true); + assert.equal(ids.has(asEventId("user-input-old")), true); + assert.equal(ids.has(asEventId("user-input-closed")), false); + assert.equal(ids.has(asEventId("user-input-tied-z-request")), true); } const windowWithPinnedRequests = yield* snapshotQuery.getThreadDetailSnapshot(threadW, { @@ -2457,12 +2481,14 @@ projectionSnapshotLayer("ProjectionSnapshotQuery windowed thread detail", (it) = }); assert.equal(windowWithPinnedRequests._tag, "Some"); if (windowWithPinnedRequests._tag === "Some") { - const ids = windowWithPinnedRequests.value.thread.activities.map((activity) => activity.id); + const ids = new Set( + windowWithPinnedRequests.value.thread.activities.map((activity) => activity.id), + ); assert.equal(windowWithPinnedRequests.value.thread.activities.length, 503); - assert.equal(ids.includes(asEventId("approval-old")), true); - assert.equal(ids.includes(asEventId("user-input-old")), true); - assert.equal(ids.includes(asEventId("user-input-closed")), false); - assert.equal(ids.includes(asEventId("user-input-tied-z-request")), true); + assert.equal(ids.has(asEventId("approval-old")), true); + assert.equal(ids.has(asEventId("user-input-old")), true); + assert.equal(ids.has(asEventId("user-input-closed")), false); + assert.equal(ids.has(asEventId("user-input-tied-z-request")), true); } }), ); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index 42db0c4e2c5d..b8b19bb285f4 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -26,6 +26,7 @@ import { ProjectId, ThreadId, ThreadAnnotation, + ThreadWorktreeCleanup, } from "@t3tools/contracts"; import * as Arr from "effect/Array"; import * as Effect from "effect/Effect"; @@ -92,6 +93,7 @@ const ProjectionThreadDbRowSchema = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), annotation: Schema.NullOr(Schema.fromJsonString(ThreadAnnotation)), + worktreeCleanup: Schema.optional(Schema.NullOr(Schema.fromJsonString(ThreadWorktreeCleanup))), }), ); const ProjectionThreadActivityDbRowSchema = ProjectionThreadActivity.mapFields( @@ -444,6 +446,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -482,6 +485,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -489,8 +493,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { has_actionable_proposed_plan AS "hasActionableProposedPlan", deleted_at AS "deletedAt" FROM projection_threads - WHERE deleted_at IS NULL - AND archived_at IS NULL + WHERE (deleted_at IS NULL AND archived_at IS NULL) + OR worktree_cleanup_json IS NOT NULL ORDER BY project_id ASC, created_at ASC, thread_id ASC `, }); @@ -522,6 +526,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -966,6 +971,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -974,8 +980,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { deleted_at AS "deletedAt" FROM projection_threads WHERE thread_id = ${threadId} - AND deleted_at IS NULL - AND archived_at IS NULL + AND ((deleted_at IS NULL AND archived_at IS NULL) + OR worktree_cleanup_json IS NOT NULL) LIMIT 1 `, }); @@ -1727,6 +1733,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), deletedAt: row.deletedAt, messages: messagesByThread.get(row.threadId) ?? [], proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], @@ -1936,6 +1943,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), latestUserMessageId: row.latestUserMessageId, + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), deletedAt: row.deletedAt, messages: [], proposedPlans: proposedPlansByThread.get(row.threadId) ?? [], @@ -2051,7 +2059,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { : Result.failVoid, ), threads: Arr.filterMap(threadRows, (row) => - row.deletedAt === null + row.deletedAt === null || row.worktreeCleanup != null ? Result.succeed({ id: row.threadId, projectId: row.projectId, @@ -2073,6 +2081,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null + ? { worktreeCleanup: row.worktreeCleanup } + : {}), session: sessionByThread.get(row.threadId) ?? null, latestUserMessageAt: row.latestUserMessageAt, hasPendingApprovals: row.pendingApprovalCount > 0, @@ -2220,6 +2231,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: row.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(row), ...(row.annotation !== null ? { annotation: row.annotation } : {}), + ...(row.worktreeCleanup != null ? { worktreeCleanup: row.worktreeCleanup } : {}), session: sessionByThread.get(row.threadId) ?? null, latestUserMessageAt: row.latestUserMessageAt, hasPendingApprovals: row.pendingApprovalCount > 0, @@ -2501,6 +2513,9 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { pinOrderKey: threadRow.value.pinOrderKey ?? null, titleRegeneration: mapTitleRegeneration(threadRow.value), ...(threadRow.value.annotation !== null ? { annotation: threadRow.value.annotation } : {}), + ...(threadRow.value.worktreeCleanup != null + ? { worktreeCleanup: threadRow.value.worktreeCleanup } + : {}), session: Option.isSome(sessionRow) ? mapSessionRow(sessionRow.value) : null, latestUserMessageAt: threadRow.value.latestUserMessageAt, hasPendingApprovals: threadRow.value.pendingApprovalCount > 0, diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 34b1b995a3ad..7a1c21cefb7e 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -1,10 +1,34 @@ -import { ThreadId } from "@t3tools/contracts"; +import { + GitCommandError, + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationCommand, + type ThreadWorktreeCleanup, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import { it as effectIt } from "@effect/vitest"; import { describe, expect, it } from "vite-plus/test"; -import { logCleanupCauseUnlessInterrupted } from "./ThreadDeletionReactor.ts"; +import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import { + ProjectionThreadRepository, + type ProjectionThread, +} from "../../persistence/Services/ProjectionThreads.ts"; +import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import * as TerminalManager from "../../terminal/Manager.ts"; +import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts"; +import { + logCleanupCauseUnlessInterrupted, + ThreadDeletionReactorLive, +} from "./ThreadDeletionReactor.ts"; describe("logCleanupCauseUnlessInterrupted", () => { const threadId = ThreadId.make("thread-deletion-reactor-test"); @@ -36,3 +60,148 @@ describe("logCleanupCauseUnlessInterrupted", () => { } }); }); + +function cleanupRow( + id: string, + cleanup: ThreadWorktreeCleanup, + deletedAt: string, +): ProjectionThread { + return { + threadId: ThreadId.make(id), + projectId: ProjectId.make("project-cleanup"), + title: id, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: id, + worktreePath: cleanup.worktreePath, + latestTurnId: null, + createdAt: deletedAt, + updatedAt: deletedAt, + archivedAt: null, + settledOverride: null, + settledAt: null, + snoozedUntil: null, + snoozedAt: null, + pinnedAt: null, + annotation: null, + worktreeCleanup: cleanup, + latestUserMessageId: null, + latestUserMessageAt: null, + pendingApprovalCount: 0, + pendingUserInputCount: 0, + hasActionableProposedPlan: 0, + deletedAt, + }; +} + +describe("durable worktree cleanup", () => { + effectIt.live("resumes same-repository cleanup in order and persists failures", () => + Effect.gen(function* () { + const root = "/repo"; + const first = cleanupRow( + "cleanup-first", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/first", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const second = cleanupRow( + "cleanup-second", + { + status: "queued", + repositoryRoot: root, + worktreePath: "/worktrees/second", + queuedAt: "2026-08-23T00:00:01.000Z", + blockedByThreadId: first.threadId, + }, + "2026-08-23T00:00:01.000Z", + ); + const third = cleanupRow( + "cleanup-third", + { + status: "queued", + repositoryRoot: root, + worktreePath: "/worktrees/third", + queuedAt: "2026-08-23T00:00:02.000Z", + blockedByThreadId: second.threadId, + }, + "2026-08-23T00:00:02.000Z", + ); + const rows = new Map([ + [first.threadId, first], + [second.threadId, second], + [third.threadId, third], + ]); + const removals: string[] = []; + const updates: Array< + Extract<OrchestrationCommand, { type: "thread.worktree-cleanup.update" }> + > = []; + + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.never, + latestSequence: Effect.succeed(0), + readEvents: () => Stream.empty, + dispatch: (command) => { + if (command.type === "thread.worktree-cleanup.update") { + updates.push(command); + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: updates.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([first, second, third]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path }) => + Effect.gen(function* () { + removals.push(path); + if (path === second.worktreePath) { + return yield* new GitCommandError({ + operation: "remove worktree", + command: "git worktree remove", + cwd: root, + detail: "permission denied", + }); + } + }), + }), + Layer.mock(ProviderService)({ stopSession: () => Effect.void }), + Layer.mock(TerminalManager.TerminalManager)({ close: () => Effect.void }), + NodeServices.layer, + ); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(dependencies), + Layer.merge(dependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* reactor.drain; + }).pipe(Effect.provide(testLayer)); + + expect(removals).toEqual(["/worktrees/first", "/worktrees/second", "/worktrees/third"]); + expect( + updates.map((command) => [command.threadId, command.cleanup?.status ?? "complete"]), + ).toEqual([ + [first.threadId, "complete"], + [second.threadId, "deleting"], + [second.threadId, "failed"], + [third.threadId, "deleting"], + [third.threadId, "complete"], + ]); + expect(updates[2]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("permission denied"), + }); + }), + ); +}); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index a026f5ad81bd..1c0505e7857d 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -1,10 +1,17 @@ -import type { OrchestrationEvent } from "@t3tools/contracts"; -import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { CommandId, type OrchestrationEvent, type ThreadWorktreeCleanup } from "@t3tools/contracts"; +import { makeDrainableWorker, type DrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as Stream from "effect/Stream"; +import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import { ProjectionThreadRepository } from "../../persistence/Services/ProjectionThreads.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; @@ -15,6 +22,11 @@ import { import { forkParked } from "../../serverActivation.ts"; type ThreadDeletedEvent = Extract<OrchestrationEvent, { type: "thread.deleted" }>; +type PendingCleanup = Exclude<ThreadWorktreeCleanup, { readonly status: "failed" }>; +type CleanupJob = { + readonly threadId: ThreadDeletedEvent["payload"]["threadId"]; + readonly cleanup: PendingCleanup; +}; export const logCleanupCauseUnlessInterrupted = <R, E>({ effect, @@ -39,8 +51,19 @@ export const logCleanupCauseUnlessInterrupted = <R, E>({ const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngineService; + const gitWorkflow = yield* GitWorkflowService; + const projectionThreads = yield* ProjectionThreadRepository; const providerService = yield* ProviderService; const terminalManager = yield* TerminalManager.TerminalManager; + const crypto = yield* Crypto.Crypto; + const cleanupWorkersRef = yield* Ref.make<ReadonlyMap<string, DrainableWorker<CleanupJob>>>( + new Map(), + ); + const enqueuedCleanupThreadIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); + + const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); + const serverCommandId = (tag: string) => + crypto.randomUUIDv4.pipe(Effect.map((uuid) => CommandId.make(`server:${tag}:${uuid}`))); const stopProviderSession = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => logCleanupCauseUnlessInterrupted({ @@ -80,20 +103,176 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processThreadDeletedSafely); + const dispatchCleanup = Effect.fn("dispatchThreadWorktreeCleanup")(function* ( + threadId: CleanupJob["threadId"], + cleanup: ThreadWorktreeCleanup | null, + ) { + yield* orchestrationEngine.dispatch({ + type: "thread.worktree-cleanup.update", + commandId: yield* serverCommandId("worktree-cleanup-update"), + threadId, + cleanup, + }); + }); + + const processCleanup = Effect.fn("processThreadWorktreeCleanup")(function* (job: CleanupJob) { + const projected = yield* projectionThreads.getById({ threadId: job.threadId }); + if (Option.isNone(projected)) return; + const current = projected.value.worktreeCleanup; + if (current == null || current.status === "failed") return; + + const startedAt = yield* nowIso; + const deleting = { + status: "deleting" as const, + repositoryRoot: current.repositoryRoot, + worktreePath: current.worktreePath, + startedAt: current.status === "deleting" ? current.startedAt : startedAt, + }; + if (current.status === "queued") { + yield* dispatchCleanup(job.threadId, deleting); + } + + const removal = yield* Effect.result( + gitWorkflow.removeWorktree({ + cwd: deleting.repositoryRoot, + path: deleting.worktreePath, + force: true, + }), + ); + if (Result.isSuccess(removal)) { + yield* dispatchCleanup(job.threadId, null); + return; + } + + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: removal.failure.message, + }); + }); + + const processCleanupSafely = (job: CleanupJob) => + processCleanup(job).pipe( + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); + const detail = Cause.pretty(cause); + return Effect.gen(function* () { + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + status: "failed", + repositoryRoot: job.cleanup.repositoryRoot, + worktreePath: job.cleanup.worktreePath, + startedAt: job.cleanup.status === "deleting" ? job.cleanup.startedAt : failedAt, + failedAt, + error: detail, + }).pipe( + Effect.catchCause((dispatchCause) => + Effect.logWarning("thread worktree cleanup failure could not be persisted", { + threadId: job.threadId, + worktreePath: job.cleanup.worktreePath, + cleanupCause: detail, + dispatchCause: Cause.pretty(dispatchCause), + }), + ), + ); + }); + }), + ); + + const removeEnqueuedCleanupThreadId = (threadId: CleanupJob["threadId"]) => + Ref.update(enqueuedCleanupThreadIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.delete(threadId); + return next; + }); + + const getCleanupWorker = Effect.fn("getThreadWorktreeCleanupWorker")(function* ( + repositoryRoot: string, + ) { + const existing = (yield* Ref.get(cleanupWorkersRef)).get(repositoryRoot); + if (existing) return existing; + const created = yield* makeDrainableWorker((job: CleanupJob) => + processCleanupSafely(job).pipe(Effect.ensuring(removeEnqueuedCleanupThreadId(job.threadId))), + ); + return yield* Ref.modify(cleanupWorkersRef, (workers) => { + const current = workers.get(repositoryRoot); + if (current) return [current, workers] as const; + const next = new Map(workers); + next.set(repositoryRoot, created); + return [created, next] as const; + }); + }); + + const enqueueCleanup = Effect.fn("enqueueThreadWorktreeCleanup")(function* (job: CleanupJob) { + const accepted = yield* Ref.modify(enqueuedCleanupThreadIdsRef, (threadIds) => { + if (threadIds.has(job.threadId)) return [false, threadIds] as const; + const next = new Set(threadIds); + next.add(job.threadId); + return [true, next] as const; + }); + if (!accepted) return; + + const cleanupWorker = yield* getCleanupWorker(job.cleanup.repositoryRoot); + yield* cleanupWorker.enqueue(job); + }); + + const enqueueCleanupFromEvent = (event: OrchestrationEvent) => { + if (event.type === "thread.deleted") { + const cleanup = event.payload.worktreeCleanup; + return cleanup == null || cleanup.status === "failed" + ? Effect.void + : enqueueCleanup({ threadId: event.payload.threadId, cleanup }); + } + if (event.type === "thread.worktree-cleanup-updated") { + const cleanup = event.payload.cleanup; + return cleanup == null || cleanup.status === "failed" + ? Effect.void + : enqueueCleanup({ threadId: event.payload.threadId, cleanup }); + } + return Effect.void; + }; + + const cleanupDrain = Effect.gen(function* () { + const workers = yield* Ref.get(cleanupWorkersRef); + yield* Effect.forEach(workers.values(), (cleanupWorker) => cleanupWorker.drain, { + concurrency: "unbounded", + }); + }); + const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () { yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { - if (event.type !== "thread.deleted") { - return Effect.void; + if (event.type === "thread.deleted") { + return Effect.all([worker.enqueue(event), enqueueCleanupFromEvent(event)]).pipe( + Effect.asVoid, + ); } - return worker.enqueue(event); + return enqueueCleanupFromEvent(event); }), ); + + yield* projectionThreads.listPendingWorktreeCleanup().pipe( + Effect.flatMap((resumable) => + Effect.forEach(resumable, (thread) => { + const cleanup = thread.worktreeCleanup; + return cleanup == null || cleanup.status === "failed" + ? Effect.void + : enqueueCleanup({ threadId: thread.threadId, cleanup }); + }), + ), + Effect.catchCause((cause) => + Effect.logWarning("thread worktree cleanup resume failed", { + cause: Cause.pretty(cause), + }), + ), + ); }); return { start, - drain: worker.drain, + drain: Effect.all([worker.drain, cleanupDrain]).pipe(Effect.asVoid), } satisfies ThreadDeletionReactorShape; }); diff --git a/apps/server/src/orchestration/Schemas.ts b/apps/server/src/orchestration/Schemas.ts index 33c5d02d3274..99123c70b96a 100644 --- a/apps/server/src/orchestration/Schemas.ts +++ b/apps/server/src/orchestration/Schemas.ts @@ -9,6 +9,7 @@ import { ThreadRuntimeModeSetPayload as ContractsThreadRuntimeModeSetPayloadSchema, ThreadInteractionModeSetPayload as ContractsThreadInteractionModeSetPayloadSchema, ThreadDeletedPayload as ContractsThreadDeletedPayloadSchema, + ThreadWorktreeCleanupUpdatedPayload as ContractsThreadWorktreeCleanupUpdatedPayloadSchema, ThreadUnarchivedPayload as ContractsThreadUnarchivedPayloadSchema, ThreadUnsettledPayload as ContractsThreadUnsettledPayloadSchema, ThreadSnoozedPayload as ContractsThreadSnoozedPayloadSchema, @@ -42,6 +43,8 @@ export const ThreadMetaUpdatedPayload = ContractsThreadMetaUpdatedPayloadSchema; export const ThreadRuntimeModeSetPayload = ContractsThreadRuntimeModeSetPayloadSchema; export const ThreadInteractionModeSetPayload = ContractsThreadInteractionModeSetPayloadSchema; export const ThreadDeletedPayload = ContractsThreadDeletedPayloadSchema; +export const ThreadWorktreeCleanupUpdatedPayload = + ContractsThreadWorktreeCleanupUpdatedPayloadSchema; export const ThreadUnarchivedPayload = ContractsThreadUnarchivedPayloadSchema; export const ThreadUnsettledPayload = ContractsThreadUnsettledPayloadSchema; export const ThreadSnoozedPayload = ContractsThreadSnoozedPayloadSchema; diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index fea36b5717fe..6235131bd92e 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -103,6 +103,10 @@ const seedReadModel = Effect.gen(function* () { }); type PlannedEvent = Omit<OrchestrationEvent, "sequence">; +type PlannedThreadDeletedEvent = Omit< + Extract<OrchestrationEvent, { type: "thread.deleted" }>, + "sequence" +>; function normalizeDeleteEvent(event: PlannedEvent | ReadonlyArray<PlannedEvent>) { const events = Array.isArray(event) ? event : [event]; @@ -137,6 +141,162 @@ function normalizeDeleteEvent(event: PlannedEvent | ReadonlyArray<PlannedEvent>) } it.layer(NodeServices.layer)("decider deletion flows", (it) => { + it.effect("persists cleanup and queues later deletions from the same repository", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread, index) => ({ + ...thread, + branch: `cleanup-${index + 1}`, + worktreePath: `/tmp/project-delete-worktrees/cleanup-${index + 1}`, + })), + }; + + const first = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-1"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }); + const firstEvent = (Array.isArray(first) ? first[0] : first) as PlannedThreadDeletedEvent; + expect(firstEvent.type).toBe("thread.deleted"); + if (firstEvent.type !== "thread.deleted") return; + expect(firstEvent.payload.worktreeCleanup).toMatchObject({ + status: "deleting", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-1", + }); + + const afterFirst = yield* projectEvent(readModel, { ...firstEvent, sequence: 4 }); + const second = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-2"), + threadId: asThreadId("thread-delete-2"), + deleteWorktree: true, + }, + readModel: afterFirst, + }); + const secondEvent = (Array.isArray(second) ? second[0] : second) as PlannedThreadDeletedEvent; + expect(secondEvent.type).toBe("thread.deleted"); + if (secondEvent.type !== "thread.deleted") return; + expect(secondEvent.payload.worktreeCleanup).toMatchObject({ + status: "queued", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-2", + blockedByThreadId: asThreadId("thread-delete-1"), + }); + }), + ); + + it.effect("refuses to delete a worktree still owned by another live thread", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => ({ + ...thread, + branch: "shared-cleanup", + worktreePath: "/tmp/project-delete-worktrees/shared-cleanup", + })), + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-shared-worktree"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }), + ); + expect(error.message).toContain("is still used by thread 'thread-delete-2'"); + }), + ); + + it.effect("retries or abandons a persisted cleanup failure", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { + ...thread, + branch: "cleanup-retry", + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + } + : thread, + ), + }; + const deleted = (yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-cleanup-retry-delete"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + })) as PlannedThreadDeletedEvent; + const afterDelete = yield* projectEvent(readModel, { ...deleted, sequence: 4 }); + const failed = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.update", + commandId: asCommandId("cmd-cleanup-failed"), + threadId: asThreadId("thread-delete-1"), + cleanup: { + status: "failed", + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + startedAt: "2026-01-01T00:00:00.000Z", + failedAt: "2026-01-01T00:00:01.000Z", + error: "permission denied", + }, + }, + readModel: afterDelete, + }); + const failedEvent = (Array.isArray(failed) ? failed[0] : failed) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + const afterFailure = yield* projectEvent(afterDelete, { ...failedEvent, sequence: 5 }); + + const retry = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.retry", + commandId: asCommandId("cmd-cleanup-retry"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFailure, + }); + const retryEvent = (Array.isArray(retry) ? retry[0] : retry) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + expect(retryEvent.payload.cleanup?.status).toBe("deleting"); + + const abandon = yield* decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.abandon", + commandId: asCommandId("cmd-cleanup-abandon"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFailure, + }); + const abandonEvent = (Array.isArray(abandon) ? abandon[0] : abandon) as Extract< + OrchestrationEvent, + { type: "thread.worktree-cleanup-updated" } + >; + expect(abandonEvent.payload.cleanup).toBeNull(); + }), + ); + it.effect("rejects deleting a non-empty project without force", () => Effect.gen(function* () { const readModel = yield* seedReadModel; @@ -154,6 +314,44 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }), ); + it.effect("rejects project deletion while a deleted thread is cleaning up its worktree", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { + ...thread, + deletedAt: "2026-01-01T00:00:01.000Z", + worktreeCleanup: { + status: "deleting" as const, + repositoryRoot: "/tmp/project-delete", + worktreePath: "/tmp/project-delete-worktrees/cleanup-1", + startedAt: "2026-01-01T00:00:01.000Z", + }, + } + : { ...thread, deletedAt: "2026-01-01T00:00:01.000Z" }, + ), + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.delete", + commandId: asCommandId("cmd-project-delete-during-cleanup"), + projectId: asProjectId("project-delete"), + force: true, + }, + readModel, + }), + ); + + expect(error.message).toContain("thread-delete-1"); + expect(error.message).toContain("Wait for cleanup to finish or keep the worktree first"); + }), + ); + it.effect("reuses thread.delete semantics when force-deleting a non-empty project", () => Effect.gen(function* () { const readModel = yield* seedReadModel; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 96016f702421..c3f7c0fa1e53 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -20,6 +20,7 @@ import { requireThreadAbsent, requireThreadNotArchived, } from "./commandInvariants.ts"; +import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; import { projectEvent } from "./projector.ts"; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); @@ -158,6 +159,47 @@ function nextAnnotationAnchorMessageId(thread: OrchestrationReadModel["threads"] return latestUserMessageId(thread) ?? thread.annotation?.anchorMessageId; } +function worktreeCleanupTimestamp( + cleanup: NonNullable<OrchestrationReadModel["threads"][number]["worktreeCleanup"]>, +): string { + switch (cleanup.status) { + case "deleting": + return cleanup.startedAt; + case "queued": + return cleanup.queuedAt; + case "failed": + return cleanup.failedAt; + } +} + +function findWorktreeCleanupBlocker( + readModel: OrchestrationReadModel, + repositoryRoot: string, + exceptThreadId?: string, +) { + const normalizedRoot = normalizeProjectPathForComparison(repositoryRoot); + return readModel.threads + .filter((candidate) => { + const cleanup = candidate.worktreeCleanup; + return ( + candidate.id !== exceptThreadId && + cleanup != null && + cleanup.status !== "failed" && + normalizeProjectPathForComparison(cleanup.repositoryRoot) === normalizedRoot + ); + }) + .toSorted((left, right) => { + const leftCleanup = left.worktreeCleanup; + const rightCleanup = right.worktreeCleanup; + if (leftCleanup == null || rightCleanup == null) return 0; + return ( + worktreeCleanupTimestamp(rightCleanup).localeCompare( + worktreeCleanupTimestamp(leftCleanup), + ) || right.id.localeCompare(left.id) + ); + })[0]; +} + function withEventBase( input: Pick<OrchestrationCommand, "commandId"> & { readonly aggregateKind: OrchestrationEvent["aggregateKind"]; @@ -320,9 +362,15 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" command, projectId: command.projectId, }); - const activeThreads = listThreadsByProjectId(readModel, command.projectId).filter( - (thread) => thread.deletedAt === null, - ); + const projectThreads = listThreadsByProjectId(readModel, command.projectId); + const cleanupThread = projectThreads.find((thread) => thread.worktreeCleanup != null); + if (cleanupThread !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Project '${command.projectId}' cannot be deleted while thread '${cleanupThread.id}' is cleaning up its worktree. Wait for cleanup to finish or keep the worktree first.`, + }); + } + const activeThreads = projectThreads.filter((thread) => thread.deletedAt === null); if (activeThreads.length > 0 && command.force !== true) { return yield* new OrchestrationCommandInvariantError({ commandType: command.type, @@ -400,12 +448,58 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" } case "thread.delete": { - yield* requireThread({ + const thread = yield* requireThread({ readModel, command, threadId: command.threadId, }); const occurredAt = yield* nowIso; + let worktreeCleanup: NonNullable< + OrchestrationReadModel["threads"][number]["worktreeCleanup"] + > | null = null; + if (command.deleteWorktree === true) { + if (thread.worktreePath === null) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not own a worktree to delete.`, + }); + } + const project = yield* requireProject({ + readModel, + command, + projectId: thread.projectId, + }); + const normalizedWorktreePath = normalizeProjectPathForComparison(thread.worktreePath); + const sharedThread = readModel.threads.find( + (candidate) => + candidate.id !== thread.id && + candidate.deletedAt === null && + candidate.worktreePath !== null && + normalizeProjectPathForComparison(candidate.worktreePath) === normalizedWorktreePath, + ); + if (sharedThread !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${thread.worktreePath}' is still used by thread '${sharedThread.id}'.`, + }); + } + const blocker = findWorktreeCleanupBlocker(readModel, project.workspaceRoot, thread.id); + worktreeCleanup = + blocker === undefined + ? { + status: "deleting", + repositoryRoot: project.workspaceRoot, + worktreePath: thread.worktreePath, + startedAt: occurredAt, + } + : { + status: "queued", + repositoryRoot: project.workspaceRoot, + worktreePath: thread.worktreePath, + queuedAt: occurredAt, + blockedByThreadId: blocker.id, + }; + } return { ...(yield* withEventBase({ aggregateKind: "thread", @@ -417,10 +511,115 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" payload: { threadId: command.threadId, deletedAt: occurredAt, + ...(worktreeCleanup === null ? {} : { worktreeCleanup }), }, }; } + case "thread.worktree-cleanup.retry": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + const cleanup = thread.worktreeCleanup; + if (thread.deletedAt === null || cleanup == null || cleanup.status !== "failed") { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not have a failed worktree cleanup to retry.`, + }); + } + const occurredAt = yield* nowIso; + const blocker = findWorktreeCleanupBlocker(readModel, cleanup.repositoryRoot, thread.id); + const nextCleanup = + blocker === undefined + ? { + status: "deleting" as const, + repositoryRoot: cleanup.repositoryRoot, + worktreePath: cleanup.worktreePath, + startedAt: occurredAt, + } + : { + status: "queued" as const, + repositoryRoot: cleanup.repositoryRoot, + worktreePath: cleanup.worktreePath, + queuedAt: occurredAt, + blockedByThreadId: blocker.id, + }; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: nextCleanup, updatedAt: occurredAt }, + }; + } + + case "thread.worktree-cleanup.abandon": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + if (thread.deletedAt === null || thread.worktreeCleanup == null) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' does not have worktree cleanup to abandon.`, + }); + } + const occurredAt = yield* nowIso; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: null, updatedAt: occurredAt }, + }; + } + + case "thread.worktree-cleanup.update": { + const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); + const current = thread.worktreeCleanup; + if (thread.deletedAt === null || current == null) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' no longer has active worktree cleanup.`, + }); + } + if ( + command.cleanup !== null && + (command.cleanup.repositoryRoot !== current.repositoryRoot || + command.cleanup.worktreePath !== current.worktreePath) + ) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Thread '${thread.id}' cleanup paths cannot change during processing.`, + }); + } + const validTransition = + (current.status === "queued" && + (command.cleanup?.status === "deleting" || command.cleanup?.status === "failed")) || + (current.status === "deleting" && + (command.cleanup === null || + command.cleanup.status === "deleting" || + command.cleanup.status === "failed")); + if (!validTransition) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Invalid worktree cleanup transition from '${current.status}' to '${command.cleanup?.status ?? "complete"}'.`, + }); + } + const occurredAt = yield* nowIso; + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: thread.id, + occurredAt, + commandId: command.commandId, + })), + type: "thread.worktree-cleanup-updated", + payload: { threadId: thread.id, cleanup: command.cleanup, updatedAt: occurredAt }, + }; + } + case "thread.archive": { yield* requireThreadNotArchived({ readModel, diff --git a/apps/server/src/orchestration/projector.test.ts b/apps/server/src/orchestration/projector.test.ts index 9a2f6c55a75d..f2d71f15441c 100644 --- a/apps/server/src/orchestration/projector.test.ts +++ b/apps/server/src/orchestration/projector.test.ts @@ -86,6 +86,7 @@ describe("orchestration projector", () => { interactionMode: "default", branch: null, worktreePath: null, + worktreeCleanup: null, latestTurn: null, createdAt: now, updatedAt: now, diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index 743c11138cf5..5c2e91255b13 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -23,6 +23,7 @@ import { ThreadArchivedPayload, ThreadCreatedPayload, ThreadDeletedPayload, + ThreadWorktreeCleanupUpdatedPayload, ThreadInteractionModeSetPayload, ThreadMetaUpdatedPayload, ThreadProposedPlanUpsertedPayload, @@ -323,6 +324,7 @@ export function projectEvent( snoozedUntil: null, snoozedAt: null, annotation: null, + worktreeCleanup: null, deletedAt: null, messages: [], activities: [], @@ -347,11 +349,28 @@ export function projectEvent( ...nextBase, threads: updateThread(nextBase.threads, payload.threadId, { deletedAt: payload.deletedAt, + worktreeCleanup: payload.worktreeCleanup ?? null, updatedAt: payload.deletedAt, }), })), ); + case "thread.worktree-cleanup-updated": + return decodeForEvent( + ThreadWorktreeCleanupUpdatedPayload, + event.payload, + event.type, + "payload", + ).pipe( + Effect.map((payload) => ({ + ...nextBase, + threads: updateThread(nextBase.threads, payload.threadId, { + worktreeCleanup: payload.cleanup, + updatedAt: payload.updatedAt, + }), + })), + ); + case "thread.archived": return decodeForEvent(ThreadArchivedPayload, event.payload, event.type, "payload").pipe( Effect.map((payload) => ({ diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index ca72c8537337..089f8ff01a48 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -10,16 +10,18 @@ import { DeleteProjectionThreadInput, GetProjectionThreadInput, ListProjectionThreadsByProjectInput, + ListPendingWorktreeCleanupThreadsInput, ProjectionThread, ProjectionThreadRepository, type ProjectionThreadRepositoryShape, } from "../Services/ProjectionThreads.ts"; -import { ModelSelection, ThreadAnnotation } from "@t3tools/contracts"; +import { ModelSelection, ThreadAnnotation, ThreadWorktreeCleanup } from "@t3tools/contracts"; const ProjectionThreadDbRow = ProjectionThread.mapFields( Struct.assign({ modelSelection: Schema.fromJsonString(ModelSelection), annotation: Schema.NullOr(Schema.fromJsonString(ThreadAnnotation)), + worktreeCleanup: Schema.NullOr(Schema.fromJsonString(ThreadWorktreeCleanup)), }), ); type ProjectionThreadDbRow = typeof ProjectionThreadDbRow.Type; @@ -53,6 +55,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id, title_regeneration_started_at, annotation_json, + worktree_cleanup_json, latest_user_message_id, latest_user_message_at, pending_approval_count, @@ -82,6 +85,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { ${row.titleRegenerationRequestId ?? null}, ${row.titleRegenerationStartedAt ?? null}, ${row.annotation === null ? null : JSON.stringify(row.annotation)}, + ${row.worktreeCleanup == null ? null : JSON.stringify(row.worktreeCleanup)}, ${row.latestUserMessageId}, ${row.latestUserMessageAt}, ${row.pendingApprovalCount}, @@ -111,6 +115,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id = excluded.title_regeneration_request_id, title_regeneration_started_at = excluded.title_regeneration_started_at, annotation_json = excluded.annotation_json, + worktree_cleanup_json = excluded.worktree_cleanup_json, latest_user_message_id = excluded.latest_user_message_id, latest_user_message_at = excluded.latest_user_message_at, pending_approval_count = excluded.pending_approval_count, @@ -147,6 +152,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -185,6 +191,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { title_regeneration_request_id AS "titleRegenerationRequestId", title_regeneration_started_at AS "titleRegenerationStartedAt", annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", latest_user_message_id AS "latestUserMessageId", latest_user_message_at AS "latestUserMessageAt", pending_approval_count AS "pendingApprovalCount", @@ -206,6 +213,47 @@ const makeProjectionThreadRepository = Effect.gen(function* () { `, }); + const listPendingWorktreeCleanupRows = SqlSchema.findAll({ + Request: ListPendingWorktreeCleanupThreadsInput, + Result: ProjectionThreadDbRow, + execute: () => + sql` + SELECT + thread_id AS "threadId", + project_id AS "projectId", + title, + model_selection_json AS "modelSelection", + runtime_mode AS "runtimeMode", + interaction_mode AS "interactionMode", + branch, + worktree_path AS "worktreePath", + latest_turn_id AS "latestTurnId", + created_at AS "createdAt", + updated_at AS "updatedAt", + archived_at AS "archivedAt", + settled_override AS "settledOverride", + settled_at AS "settledAt", + snoozed_until AS "snoozedUntil", + snoozed_at AS "snoozedAt", + pinned_at AS "pinnedAt", + pin_order_key AS "pinOrderKey", + title_regeneration_request_id AS "titleRegenerationRequestId", + title_regeneration_started_at AS "titleRegenerationStartedAt", + annotation_json AS "annotation", + worktree_cleanup_json AS "worktreeCleanup", + latest_user_message_id AS "latestUserMessageId", + latest_user_message_at AS "latestUserMessageAt", + pending_approval_count AS "pendingApprovalCount", + pending_user_input_count AS "pendingUserInputCount", + has_actionable_proposed_plan AS "hasActionableProposedPlan", + deleted_at AS "deletedAt" + FROM projection_threads + WHERE worktree_cleanup_json IS NOT NULL + AND json_extract(worktree_cleanup_json, '$.status') IN ('deleting', 'queued') + ORDER BY deleted_at ASC, thread_id ASC + `, + }); + const upsert: ProjectionThreadRepositoryShape["upsert"] = (row) => upsertProjectionThreadRow(row).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.upsert:query")), @@ -221,6 +269,14 @@ const makeProjectionThreadRepository = Effect.gen(function* () { Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.listByProjectId:query")), ); + const listPendingWorktreeCleanup: ProjectionThreadRepositoryShape["listPendingWorktreeCleanup"] = + () => + listPendingWorktreeCleanupRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadRepository.listPendingWorktreeCleanup:query"), + ), + ); + const deleteById: ProjectionThreadRepositoryShape["deleteById"] = (input) => deleteProjectionThreadRow(input).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.deleteById:query")), @@ -230,6 +286,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { upsert, getById, listByProjectId, + listPendingWorktreeCleanup, deleteById, } satisfies ProjectionThreadRepositoryShape; }); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 96d92057bb29..4cf2d211a2e0 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -58,6 +58,7 @@ import Migration0042 from "./Migrations/042_ProjectionThreadAnnotation.ts"; import Migration0043 from "./Migrations/043_UpdateDrain.ts"; import Migration0044 from "./Migrations/044_UpdateDrainClaim.ts"; import Migration0045 from "./Migrations/045_ProjectionTurnRequestCorrelations.ts"; +import Migration0046 from "./Migrations/046_ProjectionThreadWorktreeCleanup.ts"; /** * Migration loader with all migrations defined inline. @@ -115,6 +116,7 @@ export const migrationEntries = [ [43, "UpdateDrain", Migration0043], [44, "UpdateDrainClaim", Migration0044], [45, "ProjectionTurnRequestCorrelations", Migration0045], + [46, "ProjectionThreadWorktreeCleanup", Migration0046], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts new file mode 100644 index 000000000000..765a9605684b --- /dev/null +++ b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.test.ts @@ -0,0 +1,55 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("046_ProjectionThreadWorktreeCleanup", (it) => { + it.effect("adds nullable cleanup state without changing existing rows", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 44 }); + yield* sql` + INSERT INTO projection_threads ( + thread_id, + project_id, + title, + model_selection_json, + runtime_mode, + interaction_mode, + created_at, + updated_at + ) + VALUES ( + 'thread-before-cleanup', + 'project-1', + 'Existing thread', + '{"instanceId":"codex","model":"gpt-5-codex"}', + 'full-access', + 'default', + '2026-08-23T00:00:00.000Z', + '2026-08-23T00:00:00.000Z' + ) + `; + + yield* runMigrations({ toMigrationInclusive: 46 }); + + const columns = yield* sql<{ readonly name: string; readonly notnull: number }>` + PRAGMA table_info(projection_threads) + `; + const cleanupJson = columns.find((column) => column.name === "worktree_cleanup_json"); + assert.equal(cleanupJson?.notnull, 0); + + const rows = yield* sql<{ readonly cleanup: string | null }>` + SELECT worktree_cleanup_json AS cleanup + FROM projection_threads + WHERE thread_id = 'thread-before-cleanup' + `; + assert.equal(rows[0]?.cleanup, null); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts new file mode 100644 index 000000000000..5926e05b1e47 --- /dev/null +++ b/apps/server/src/persistence/Migrations/046_ProjectionThreadWorktreeCleanup.ts @@ -0,0 +1,16 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + + if (!columns.some((column) => column.name === "worktree_cleanup_json")) { + yield* sql` + ALTER TABLE projection_threads + ADD COLUMN worktree_cleanup_json TEXT + `; + } +}); diff --git a/apps/server/src/persistence/Services/ProjectionThreads.ts b/apps/server/src/persistence/Services/ProjectionThreads.ts index b5f450c1841a..a15792ef4bf8 100644 --- a/apps/server/src/persistence/Services/ProjectionThreads.ts +++ b/apps/server/src/persistence/Services/ProjectionThreads.ts @@ -16,6 +16,7 @@ import { ProviderInteractionMode, RuntimeMode, ThreadAnnotation, + ThreadWorktreeCleanup, ThreadId, TurnId, } from "@t3tools/contracts"; @@ -48,6 +49,7 @@ export const ProjectionThread = Schema.Struct({ titleRegenerationRequestId: Schema.optional(Schema.NullOr(CommandId)), titleRegenerationStartedAt: Schema.optional(Schema.NullOr(IsoDateTime)), annotation: Schema.NullOr(ThreadAnnotation), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), latestUserMessageId: Schema.NullOr(MessageId), latestUserMessageAt: Schema.NullOr(IsoDateTime), pendingApprovalCount: NonNegativeInt, @@ -70,6 +72,7 @@ export type DeleteProjectionThreadInput = typeof DeleteProjectionThreadInput.Typ export const ListProjectionThreadsByProjectInput = Schema.Struct({ projectId: ProjectId, }); +export const ListPendingWorktreeCleanupThreadsInput = Schema.Void; export type ListProjectionThreadsByProjectInput = typeof ListProjectionThreadsByProjectInput.Type; /** @@ -99,6 +102,11 @@ export interface ProjectionThreadRepositoryShape { input: ListProjectionThreadsByProjectInput, ) => Effect.Effect<ReadonlyArray<ProjectionThread>, ProjectionRepositoryError>; + readonly listPendingWorktreeCleanup: () => Effect.Effect< + ReadonlyArray<ProjectionThread>, + ProjectionRepositoryError + >; + /** * Soft-delete a projected thread row by id. */ diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b523600b47eb..ad753f5c4ad0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -637,7 +637,6 @@ const makeWsRpcLayer = ( projectId: event.payload.projectId, }), ); - case "thread.deleted": case "thread.archived": return Effect.succeed( Option.some({ @@ -646,6 +645,8 @@ const makeWsRpcLayer = ( threadId: event.payload.threadId, }), ); + case "thread.deleted": + return threadUpsertOrRemove(event.payload.threadId, event.sequence); case "thread.unarchived": return threadUpsertOrRemove(event.payload.threadId, event.sequence); default: diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index 8e37ca7e57db..f920c7b25916 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -224,7 +224,11 @@ import { type ProviderInstanceEntry, } from "../providerInstances"; import { getTriggerDisplayModelLabel } from "./chat/providerIconUtils"; -import { SidebarThreadHoverContent } from "./sidebar/SidebarThreadHoverContent"; +import { + SidebarThreadCleanupHoverContent, + SidebarThreadHoverContent, +} from "./sidebar/SidebarThreadHoverContent"; +import { WorktreeCleanupFailureDialog } from "./WorktreeCleanupFailureDialog"; import { buildPhysicalToLogicalProjectKeyMap, buildSidebarProjectSnapshots, @@ -407,6 +411,10 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr } = props; const threadRef = scopeThreadRef(thread.environmentId, thread.id); const threadKey = scopedThreadKey(threadRef); + const cleanup = thread.worktreeCleanup ?? null; + const isCleanupPending = cleanup?.status === "deleting" || cleanup?.status === "queued"; + const isCleanupFailed = cleanup?.status === "failed"; + const [cleanupFailureOpen, setCleanupFailureOpen] = useState(false); const lastVisitedAt = useUiStateStore((state) => state.threadLastVisitedAtById[threadKey]); const isSelected = useThreadSelectionStore((state) => state.selectedThreadKeys.has(threadKey)); const runningTerminalIds = useThreadRunningTerminalIds({ @@ -498,6 +506,11 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr const terminalStatus = terminalStatusFromRunningIds(runningTerminalIds); const isConfirmingArchive = confirmingArchiveThreadKey === threadKey && !isThreadRunning; const hasActiveAnnotation = thread.annotation?.resolvedAt === null; + const cleanupBlockerTitle = + cleanup?.status === "queued" + ? (readThreadShell(scopeThreadRef(thread.environmentId, cleanup.blockedByThreadId))?.title ?? + null) + : null; const branchMismatch = resolveLocalCheckoutBranchMismatch({ effectiveEnvMode: thread.worktreePath === null ? "local" : "worktree", activeWorktreePath: thread.worktreePath, @@ -530,8 +543,13 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr terminalProcessCount={runningTerminalIds.length} terminalStatus={terminalStatus} thread={thread} + cleanupBlockerTitle={cleanupBlockerTitle} + showCleanup={!hasActiveAnnotation} /> ); + const cleanupHoverDetails = ( + <SidebarThreadCleanupHoverContent thread={thread} blockerTitle={cleanupBlockerTitle} /> + ); const threadMetaClassName = isConfirmingArchive ? "pointer-events-none opacity-0" : !isThreadRunning @@ -565,12 +583,22 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr ); const handleRowClick = useCallback( (event: React.MouseEvent) => { + if (isCleanupFailed) { + event.preventDefault(); + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) { + event.preventDefault(); + return; + } handleThreadClick(event, threadRef, orderedProjectThreadKeys); }, - [handleThreadClick, orderedProjectThreadKeys, threadRef], + [handleThreadClick, isCleanupFailed, isCleanupPending, orderedProjectThreadKeys, threadRef], ); const handleRowDoubleClick = useCallback( (event: React.MouseEvent) => { + if (cleanup !== null) return; // Already renaming this row: a double-click on the row chrome (outside the // input) must not restart and discard the in-progress edit. if (renamingThreadKey === threadKey) return; @@ -585,19 +613,25 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr event.preventDefault(); startThreadRename(threadKey, thread.title); }, - [isMobile, renamingThreadKey, startThreadRename, threadKey, thread.title], + [cleanup, isMobile, renamingThreadKey, startThreadRename, threadKey, thread.title], ); const handleRowKeyDown = useCallback( (event: React.KeyboardEvent) => { if (event.key !== "Enter" && event.key !== " ") return; event.preventDefault(); + if (isCleanupFailed) { + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) return; navigateToThread(threadRef); }, - [navigateToThread, threadRef], + [isCleanupFailed, isCleanupPending, navigateToThread, threadRef], ); const handleRowContextMenu = useCallback( (event: React.MouseEvent) => { event.preventDefault(); + if (cleanup !== null) return; const hasSelection = useThreadSelectionStore.getState().hasSelection(); if (hasSelection && isSelected) { void (async () => { @@ -643,7 +677,14 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr } })(); }, - [clearSelection, handleMultiSelectContextMenu, handleThreadContextMenu, isSelected, threadRef], + [ + cleanup, + clearSelection, + handleMultiSelectContextMenu, + handleThreadContextMenu, + isSelected, + threadRef, + ], ); const handlePrClick = useCallback( (event: React.MouseEvent<HTMLAnchorElement>) => { @@ -770,17 +811,18 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr size="sm" isActive={isActive} data-testid={`thread-row-${thread.id}`} + aria-disabled={isCleanupPending || undefined} className={`${resolveThreadRowClassName({ isActive, isSelected, - })} relative isolate`} + })} relative isolate ${isCleanupPending ? "cursor-not-allowed opacity-65" : ""}`} onClick={handleRowClick} onDoubleClick={handleRowDoubleClick} onKeyDown={handleRowKeyDown} onContextMenu={handleRowContextMenu} > <div className="flex min-w-0 flex-1 items-center gap-1.5 text-left"> - {prStatus && ( + {cleanup === null && prStatus && ( <Tooltip> <TooltipTrigger render={ @@ -824,7 +866,7 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr )} </div> <div className="ml-auto flex shrink-0 items-center gap-1.5"> - {discoveredPorts.length > 0 && ( + {cleanup === null && discoveredPorts.length > 0 && ( <Tooltip> <TooltipTrigger render={ @@ -881,7 +923,7 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr > Confirm </button> - ) : !isThreadRunning ? ( + ) : !isThreadRunning && cleanup === null ? ( appSettingsConfirmThreadArchive ? ( <div className="pointer-events-none absolute top-1/2 right-0.5 -translate-y-1/2 opacity-0 transition-opacity duration-150 max-sm:pointer-events-auto max-sm:opacity-100 group-hover/menu-sub-item:pointer-events-auto group-hover/menu-sub-item:opacity-100 group-focus-within/menu-sub-item:pointer-events-auto group-focus-within/menu-sub-item:opacity-100"> <button @@ -949,6 +991,7 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr onResolve={() => onResolveAnnotation(thread)} rowActive={threadRowActive} threadDetails={threadHoverDetails} + trailingContent={cleanupHoverDetails} threadRef={threadRef} trigger={ <span @@ -983,6 +1026,7 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr onResolve={() => onResolveAnnotation(thread)} rowActive={threadRowActive} threadDetails={threadHoverDetails} + trailingContent={cleanupHoverDetails} threadRef={threadRef} trigger={ <span @@ -1030,6 +1074,11 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr {threadHoverDetails} </TooltipPopup> </Tooltip> + <WorktreeCleanupFailureDialog + thread={thread} + open={cleanupFailureOpen} + onOpenChange={setCleanupFailureOpen} + /> </SidebarMenuSubItem> ); }); @@ -3748,9 +3797,9 @@ export default function LegacySidebar() { ? projectThreads : projectThreads.slice(0, sidebarThreadPreviewCount); const renderedThreads = pinnedCollapsedThread ? [pinnedCollapsedThread] : previewThreads; - return renderedThreads.map((thread) => - scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id)), - ); + return renderedThreads + .filter((thread) => thread.worktreeCleanup == null) + .map((thread) => scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id))); }), [ sidebarThreadSortOrder, diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 8f88f4740b16..9b7c0fa8b9ae 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -707,6 +707,49 @@ describe("resolveSidebarThreadStatus", () => { const idle = { hasPendingApprovals: false, hasPendingUserInput: false }; + it("prioritizes durable worktree cleanup over agent status", () => { + const cleanupBase = { + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/one", + }; + expect( + resolveSidebarThreadStatus({ + ...idle, + session, + worktreeCleanup: { + ...cleanupBase, + status: "deleting", + startedAt: "2026-03-09T10:00:00.000Z", + }, + }), + ).toBe("cleanup-deleting"); + expect( + resolveSidebarThreadStatus({ + ...idle, + session, + worktreeCleanup: { + ...cleanupBase, + status: "queued", + queuedAt: "2026-03-09T10:00:00.000Z", + blockedByThreadId: ThreadId.make("blocker"), + }, + }), + ).toBe("cleanup-queued"); + expect( + resolveSidebarThreadStatus({ + ...idle, + session, + worktreeCleanup: { + ...cleanupBase, + status: "failed", + startedAt: "2026-03-09T10:00:00.000Z", + failedAt: "2026-03-09T10:01:00.000Z", + error: "permission denied", + }, + }), + ).toBe("cleanup-failed"); + }); + it("prioritizes approval over a running session", () => { expect(resolveSidebarThreadStatus({ ...idle, hasPendingApprovals: true, session })).toBe( "approval", @@ -1108,6 +1151,23 @@ describe("resolveThreadStatusPill", () => { }, }; + it("uses the deleting labels and colors for cleanup tombstones", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/two", + queuedAt: "2026-03-09T10:00:00.000Z", + blockedByThreadId: ThreadId.make("blocker"), + }, + }, + }), + ).toMatchObject({ label: "Deleting (Queued)", pulse: false }); + }); + it("shows pending approval before all other statuses", () => { expect( resolveThreadStatusPill({ @@ -1430,6 +1490,41 @@ describe("getFallbackThreadIdAfterDelete", () => { expect(fallbackThreadId).toBe(ThreadId.make("thread-next")); }); + + it("skips cleanup tombstones left by an earlier delete", () => { + const fallbackThreadId = getFallbackThreadIdAfterDelete({ + threads: [ + makeThread({ + id: ThreadId.make("thread-active"), + projectId: ProjectId.make("project-1"), + createdAt: "2026-03-09T10:05:00.000Z", + messages: [], + }), + makeThread({ + id: ThreadId.make("thread-cleanup"), + projectId: ProjectId.make("project-1"), + createdAt: "2026-03-09T10:10:00.000Z", + messages: [], + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: "2026-03-09T10:10:00.000Z", + }, + }), + makeThread({ + id: ThreadId.make("thread-next"), + projectId: ProjectId.make("project-1"), + createdAt: "2026-03-09T10:07:00.000Z", + messages: [], + }), + ], + deletedThreadId: ThreadId.make("thread-active"), + sortOrder: "created_at", + }); + + expect(fallbackThreadId).toBe(ThreadId.make("thread-next")); + }); }); describe("sortProjectsForSidebar", () => { it("sorts projects by the most recent user message across their threads", () => { diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index 8e6def581d51..8bd861038c40 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -133,7 +133,10 @@ export interface ThreadStatusPill { | "Completed" | "Pending Approval" | "Awaiting Input" - | "Plan Ready"; + | "Plan Ready" + | "Deleting" + | "Deleting (Queued)" + | "Cleanup failed"; colorClass: string; dotClass: string; pulse: boolean; @@ -151,6 +154,9 @@ const THREAD_STATUS_PRIORITY: Record<ThreadStatusPill["label"], number> = { "Plan Ready": 3, Monitoring: 2, Completed: 1, + Deleting: 7, + "Deleting (Queued)": 7, + "Cleanup failed": 8, }; type ThreadStatusInput = Pick< @@ -163,6 +169,7 @@ type ThreadStatusInput = Pick< | "session" | "backgroundLiveness" | "actionResume" + | "worktreeCleanup" > & { lastVisitedAt?: string | undefined; }; @@ -476,14 +483,25 @@ export type SidebarThreadStatus = | "waiting" | "monitoring" | "failed" + | "cleanup-deleting" + | "cleanup-queued" + | "cleanup-failed" | "ready"; type SidebarThreadStatusInput = Pick< SidebarThreadSummary, - "hasPendingApprovals" | "hasPendingUserInput" | "session" | "backgroundLiveness" | "actionResume" + | "hasPendingApprovals" + | "hasPendingUserInput" + | "session" + | "backgroundLiveness" + | "actionResume" + | "worktreeCleanup" >; export function resolveSidebarThreadStatus(thread: SidebarThreadStatusInput): SidebarThreadStatus { + if (thread.worktreeCleanup?.status === "failed") return "cleanup-failed"; + if (thread.worktreeCleanup?.status === "queued") return "cleanup-queued"; + if (thread.worktreeCleanup?.status === "deleting") return "cleanup-deleting"; if (thread.hasPendingApprovals) { return "approval"; } @@ -654,6 +672,33 @@ export function resolveThreadStatusPill(input: { }): ThreadStatusPill | null { const { thread } = input; + if (thread.worktreeCleanup?.status === "failed") { + return { + label: "Cleanup failed", + colorClass: "text-red-700 dark:text-red-300", + dotClass: "bg-red-600 dark:bg-red-300", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "queued") { + return { + label: "Deleting (Queued)", + colorClass: "text-orange-700 dark:text-orange-300", + dotClass: "bg-orange-500 dark:bg-orange-300", + pulse: false, + }; + } + + if (thread.worktreeCleanup?.status === "deleting") { + return { + label: "Deleting", + colorClass: "text-orange-700 dark:text-orange-300", + dotClass: "bg-orange-500 dark:bg-orange-300", + pulse: false, + }; + } + if (thread.hasPendingApprovals) { return { label: "Pending Approval", @@ -816,7 +861,8 @@ export function getVisibleThreadsForProject<T extends Pick<Thread, "id">>(input: } export function getFallbackThreadIdAfterDelete< - T extends Pick<Thread, "id" | "projectId" | "createdAt" | "updatedAt"> & ThreadSortInput, + T extends Pick<Thread, "id" | "projectId" | "createdAt" | "updatedAt"> & + ThreadSortInput & { readonly worktreeCleanup?: unknown }, >(input: { threads: readonly T[]; deletedThreadId: T["id"]; @@ -835,6 +881,7 @@ export function getFallbackThreadIdAfterDelete< (thread) => thread.projectId === deletedThread.projectId && thread.id !== deletedThreadId && + thread.worktreeCleanup == null && !deletedThreadIds?.has(thread.id), ), sortOrder, diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index a6927b102917..498d90cf6d4d 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -104,7 +104,7 @@ import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; import { useLocalStorage } from "../hooks/useLocalStorage"; import { useNowMinute } from "../hooks/useNowMinute"; import { useEnvironments, usePrimaryEnvironmentId } from "../state/environments"; -import { useProjects, useThreadShells } from "../state/entities"; +import { readThreadShell, useProjects, useThreadShells } from "../state/entities"; import { environmentServerConfigsAtom, primaryServerKeybindingsAtom } from "../state/server"; import { vcsEnvironment } from "../state/vcs"; import { threadEnvironment } from "../state/threads"; @@ -166,6 +166,7 @@ import { SidebarThreadHoverContent, type SidebarThreadHoverContentProps, } from "./sidebar/SidebarThreadHoverContent"; +import { WorktreeCleanupFailureDialog } from "./WorktreeCleanupFailureDialog"; import { getTriggerDisplayModelLabel } from "./chat/providerIconUtils"; import { deriveProviderEntriesByEnvironment, @@ -672,6 +673,10 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { [thread.environmentId, thread.id], ); const threadKey = scopedThreadKey(threadRef); + const cleanup = thread.worktreeCleanup ?? null; + const isCleanupPending = cleanup?.status === "deleting" || cleanup?.status === "queued"; + const isCleanupFailed = cleanup?.status === "failed"; + const [cleanupFailureOpen, setCleanupFailureOpen] = useState(false); const isRegeneratingTitle = thread.titleRegeneration != null; const lastVisitedAt = useUiStateStore((state) => state.threadLastVisitedAtById[threadKey]); const isSelected = useThreadSelectionStore((state) => state.selectedThreadKeys.has(threadKey)); @@ -760,69 +765,89 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { status === "monitoring" || status === "waiting" || status === "approval" || - status === "input"; + status === "input" || + status === "cleanup-deleting" || + status === "cleanup-queued"; const shouldRecede = (status === "ready" || isInFlight) && !isUnread && !isWoke && !props.isActive && !isSelected; // Status hues follow the system-wide convention set by sidebar v1 and the // mobile Live Activity/widgets (amber approval, indigo input, sky working) // so a thread reads the same color everywhere it surfaces. const topStatus = - status === "working" + status === "cleanup-deleting" ? { - label: "Working", - icon: "working" as const, - // No shimmer: a label that animates forever is noise in a sidebar - // full of them (and repaints every vsync on high-refresh displays). - // Working is a background state, so it rests at the dim end of what - // the old pulse cycled through; only the thread you have open gets - // the label at full strength. - className: cn("text-sky-600 dark:text-sky-400", !props.isActive && "opacity-75"), + label: "Deleting", + icon: "cleanup" as const, + className: "text-orange-700 dark:text-orange-300", } - : status === "monitoring" + : status === "cleanup-queued" ? { - // Monitoring is calm background presence, not active progress - // (monitoring-pill D6), so it keeps the label at full strength. - label: "Monitoring", - icon: null, - className: "text-sky-600 dark:text-sky-400", + label: "Deleting (Queued)", + icon: "cleanup" as const, + className: "text-orange-700 dark:text-orange-300", } - : status === "waiting" + : status === "cleanup-failed" ? { - label: "Waiting", - icon: "waiting" as const, - className: "text-yellow-700 dark:text-yellow-300", + label: "Cleanup failed", + icon: null, + className: "text-red-700 dark:text-red-300", } - : status === "approval" + : status === "working" ? { - label: "Approval", - icon: null, - className: "text-amber-700 dark:text-amber-300", + label: "Working", + icon: "working" as const, + // No shimmer: a label that animates forever is noise in a sidebar + // full of them (and repaints every vsync on high-refresh displays). + // Working is a background state, so it rests at the dim end of what + // the old pulse cycled through; only the thread you have open gets + // the label at full strength. + className: cn("text-sky-600 dark:text-sky-400", !props.isActive && "opacity-75"), } - : status === "input" + : status === "monitoring" ? { - label: "Input", + // Monitoring is calm background presence, not active progress + // (monitoring-pill D6), so it keeps the label at full strength. + label: "Monitoring", icon: null, - className: "text-indigo-600 dark:text-indigo-300", + className: "text-sky-600 dark:text-sky-400", } - : status === "failed" + : status === "waiting" ? { - label: "Failed", - icon: null, - className: "text-red-700 dark:text-red-300", + label: "Waiting", + icon: "waiting" as const, + className: "text-yellow-700 dark:text-yellow-300", } - : isWoke + : status === "approval" ? { - label: "Woke", - icon: "woke" as const, + label: "Approval", + icon: null, className: "text-amber-700 dark:text-amber-300", } - : isUnread + : status === "input" ? { - label: "Done", - icon: "done" as const, - className: "text-emerald-700 dark:text-emerald-300", + label: "Input", + icon: null, + className: "text-indigo-600 dark:text-indigo-300", } - : null; + : status === "failed" + ? { + label: "Failed", + icon: null, + className: "text-red-700 dark:text-red-300", + } + : isWoke + ? { + label: "Woke", + icon: "woke" as const, + className: "text-amber-700 dark:text-amber-300", + } + : isUnread + ? { + label: "Done", + icon: "done" as const, + className: "text-emerald-700 dark:text-emerald-300", + } + : null; const isWokeStatus = topStatus?.icon === "woke"; const branchMismatch = resolveLocalCheckoutBranchMismatch({ @@ -872,6 +897,11 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { const isRemote = props.currentEnvironmentId !== null && thread.environmentId !== props.currentEnvironmentId; + const cleanupBlockerTitle = + cleanup?.status === "queued" + ? (readThreadShell(scopeThreadRef(thread.environmentId, cleanup.blockedByThreadId))?.title ?? + null) + : null; const detailsTooltip = ( <SidebarThreadTooltip @@ -887,14 +917,24 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { branchMismatch={branchMismatch} terminalStatus={terminalStatus} terminalProcessCount={terminalProcessCount} + cleanupBlockerTitle={cleanupBlockerTitle} /> ); const handleClick = useCallback( (event: ReactMouseEvent) => { + if (isCleanupFailed) { + event.preventDefault(); + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) { + event.preventDefault(); + return; + } onThreadClick(event, threadRef); }, - [onThreadClick, threadRef], + [isCleanupFailed, isCleanupPending, onThreadClick, threadRef], ); const handleAcknowledgeWokeClick = useCallback( (event: ReactMouseEvent) => { @@ -908,21 +948,28 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { const handleContextMenu = useCallback( (event: ReactMouseEvent) => { event.preventDefault(); + if (cleanup !== null) return; onContextMenu(threadRef, { x: event.clientX, y: event.clientY }); }, - [onContextMenu, threadRef], + [cleanup, onContextMenu, threadRef], ); const handleKeyDown = useCallback( (event: ReactKeyboardEvent) => { if (event.target !== event.currentTarget) return; if (event.key !== "Enter" && event.key !== " ") return; event.preventDefault(); + if (isCleanupFailed) { + setCleanupFailureOpen(true); + return; + } + if (isCleanupPending) return; onThreadActivate(threadRef); }, - [onThreadActivate, threadRef], + [isCleanupFailed, isCleanupPending, onThreadActivate, threadRef], ); const handleDoubleClick = useCallback( (event: ReactMouseEvent) => { + if (cleanup !== null) return; if (isRenaming || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) { return; } @@ -930,7 +977,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { event.preventDefault(); onStartRename(threadRef, thread.title); }, - [isRenaming, onStartRename, thread.title, threadRef], + [cleanup, isRenaming, onStartRename, thread.title, threadRef], ); const renameCommittedRef = useRef(false); useEffect(() => { @@ -1001,7 +1048,9 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // Snooze is offered only where it can succeed: capability-gated and never // on blocked-on-you work or queued turns (the server rejects both). const showSnoozeButton = - props.snoozeSupported && canSnooze(thread, { now: new Date().toISOString() }); + cleanup === null && + props.snoozeSupported && + canSnooze(thread, { now: new Date().toISOString() }); // If the thread becomes blocked while the popover is open, the button // unmounts without firing onOpenChange(false). Deriving the flag keeps a // stale true from permanently hiding the status label / pinning the @@ -1043,6 +1092,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { !props.isActive && !isSelected && "opacity-70 transition-opacity hover:opacity-100", + isCleanupPending && "cursor-not-allowed opacity-65 hover:opacity-65", ); const title = isRenaming ? ( @@ -1092,7 +1142,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { // A real link so cmd/ctrl+click and middle-click open the host in the // browser. A plain click still opens T3's pull request view. const prBadge = - prStatus && pr ? ( + cleanup === null && prStatus && pr ? ( <a href={pr.url} target="_blank" @@ -1125,7 +1175,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { </span> ) : null; const pinIndicator = props.isPinned ? ( - props.pinningSupported ? ( + props.pinningSupported && cleanup === null ? ( <Tooltip> <TooltipTrigger render={ @@ -1163,6 +1213,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { role="button" tabIndex={0} data-testid="sidebar-row-slim" + aria-disabled={isCleanupPending || undefined} aria-busy={isRegeneratingTitle || undefined} className={cn(rowSurfaceClassName, "flex h-9 items-center gap-2.5 px-2.5")} onClick={handleClick} @@ -1292,13 +1343,18 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { </TooltipTrigger> {detailsTooltip} </Tooltip> + <WorktreeCleanupFailureDialog + thread={thread} + open={cleanupFailureOpen} + onOpenChange={setCleanupFailureOpen} + /> </li> ); } const diff = latestTurnDiff(thread); - const sortable = props.sortable; + const sortable = cleanup === null ? props.sortable : undefined; return ( <li data-thread-item @@ -1324,6 +1380,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { role="button" tabIndex={0} data-testid="sidebar-row-card" + aria-disabled={isCleanupPending || undefined} aria-busy={isRegeneratingTitle || undefined} className={rowSurfaceClassName} onClick={handleClick} @@ -1448,7 +1505,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { topStatus.className, )} > - {topStatus.icon === "working" ? ( + {topStatus.icon === "working" || topStatus.icon === "cleanup" ? ( <CircleDashedIcon aria-hidden className="size-4 shrink-0" /> ) : topStatus.icon === "waiting" ? ( <span @@ -1466,6 +1523,10 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { <span aria-hidden> <WorkingDuration startedAt={resolveWorkingStartedAt(thread)} /> </span> + ) : status === "cleanup-deleting" && cleanup?.status === "deleting" ? ( + <span aria-hidden> + <WorkingDuration startedAt={cleanup.startedAt} /> + </span> ) : null} </span> ) @@ -1473,7 +1534,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { threadTimeLabel(thread) )} </span> - {props.settlementSupported || showSnoozeButton ? ( + {(cleanup === null && props.settlementSupported) || showSnoozeButton ? ( <span className={cn( // focus-visible, not focus-within: a mouse click leaves @@ -1493,7 +1554,7 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { timestampFormat={props.timestampFormat} /> ) : null} - {props.settlementSupported ? ( + {cleanup === null && props.settlementSupported ? ( <Tooltip> <TooltipTrigger render={ @@ -1576,6 +1637,11 @@ const SidebarThreadRow = memo(function SidebarThreadRow(props: { </TooltipTrigger> {detailsTooltip} </Tooltip> + <WorktreeCleanupFailureDialog + thread={thread} + open={cleanupFailureOpen} + onOpenChange={setCleanupFailureOpen} + /> </li> ); }); @@ -2229,9 +2295,9 @@ export default function Sidebar() { ); const orderedThreadKeys = useMemo( () => - orderedThreads.map((thread) => - scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id)), - ), + orderedThreads + .filter((thread) => thread.worktreeCleanup == null) + .map((thread) => scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id))), [orderedThreads], ); // Rows call back into the click handler without carrying the ordered list as diff --git a/apps/web/src/components/WorktreeCleanupFailureDialog.tsx b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx new file mode 100644 index 000000000000..8277d5386374 --- /dev/null +++ b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx @@ -0,0 +1,97 @@ +import type { SidebarThreadSummary } from "../types"; +import { threadEnvironment } from "../state/threads"; +import { useAtomCommand } from "../state/use-atom-command"; +import { Button } from "./ui/button"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "./ui/dialog"; + +export function WorktreeCleanupFailureDialog(props: { + thread: SidebarThreadSummary; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const retry = useAtomCommand(threadEnvironment.retryWorktreeCleanup, { reportFailure: true }); + const abandon = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { + reportFailure: true, + }); + const cleanup = props.thread.worktreeCleanup; + if (cleanup?.status !== "failed") return null; + + const details = [ + `Thread: ${props.thread.id} — ${props.thread.title}`, + `Worktree: ${cleanup.worktreePath}`, + `Repository: ${cleanup.repositoryRoot}`, + `Failed: ${cleanup.failedAt}`, + "", + cleanup.error, + ].join("\n"); + + return ( + <Dialog open={props.open} onOpenChange={props.onOpenChange}> + <DialogPopup className="max-w-xl"> + <DialogHeader> + <DialogTitle>Worktree cleanup failed</DialogTitle> + <DialogDescription> + The thread is deleted, but LastCode has not removed its worktree yet. + </DialogDescription> + </DialogHeader> + <DialogPanel className="space-y-3"> + <div className="text-sm"> + <div className="font-medium">{props.thread.title}</div> + <div className="font-mono text-xs text-muted-foreground">{props.thread.id}</div> + </div> + <div className="rounded-lg border border-border/70 bg-muted/40 p-3"> + <div className="mb-1 text-xs font-medium text-muted-foreground">Worktree</div> + <div className="break-all font-mono text-xs">{cleanup.worktreePath}</div> + </div> + <pre className="max-h-48 overflow-auto whitespace-pre-wrap rounded-lg border border-red-500/25 bg-red-500/8 p-3 text-xs text-red-800 dark:text-red-200"> + {cleanup.error} + </pre> + </DialogPanel> + <DialogFooter className="sm:flex-wrap"> + <Button + type="button" + variant="destructive-outline" + onClick={() => { + void abandon({ + environmentId: props.thread.environmentId, + input: { threadId: props.thread.id }, + }).then((result) => { + if (result._tag === "Success") props.onOpenChange(false); + }); + }} + > + Keep worktree + </Button> + <Button + type="button" + variant="outline" + onClick={() => void navigator.clipboard.writeText(details)} + > + Copy details + </Button> + <Button + type="button" + onClick={() => { + void retry({ + environmentId: props.thread.environmentId, + input: { threadId: props.thread.id }, + }).then((result) => { + if (result._tag === "Success") props.onOpenChange(false); + }); + }} + > + Retry + </Button> + </DialogFooter> + </DialogPopup> + </Dialog> + ); +} diff --git a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx index 4826863b5ac5..d31f66beee11 100644 --- a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx +++ b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx @@ -23,6 +23,8 @@ export interface SidebarThreadHoverContentProps { } | null; terminalStatus: TerminalStatusIndicator | null; terminalProcessCount: number; + cleanupBlockerTitle?: string | null; + showCleanup?: boolean; } function terminalProcessLabel(count: number): string { @@ -109,6 +111,41 @@ export function SidebarThreadHoverContent(props: SidebarThreadHoverContentProps) </div> ) : null} </div> + {props.showCleanup === false ? null : ( + <SidebarThreadCleanupHoverContent + thread={props.thread} + blockerTitle={props.cleanupBlockerTitle ?? null} + /> + )} + </div> + ); +} + +export function SidebarThreadCleanupHoverContent(props: { + thread: SidebarThreadSummary; + blockerTitle: string | null; +}) { + const cleanup = props.thread.worktreeCleanup; + if (cleanup == null || cleanup.status === "failed") return null; + + return ( + <div className="-mx-[var(--floating-content-inset)] -mb-[var(--floating-content-inset)] border-t border-orange-600/25 bg-orange-400 px-[var(--floating-content-inset)] py-2 text-xs text-foreground dark:bg-orange-400 dark:text-background"> + {cleanup.status === "deleting" ? ( + <> + <div className="font-medium">Deleting worktree</div> + <div className="mt-1 break-all font-mono text-[11px] opacity-80"> + {cleanup.worktreePath} + </div> + </> + ) : ( + <> + <div className="font-medium">Waiting for cleanup</div> + <div className="mt-1 truncate"> + {cleanup.blockedByThreadId} + {props.blockerTitle ? ` — ${props.blockerTitle}` : ""} + </div> + </> + )} </div> ); } diff --git a/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx b/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx index e9ded512f939..083b75d03896 100644 --- a/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx +++ b/apps/web/src/components/thread-annotation/ThreadAnnotation.tsx @@ -268,6 +268,7 @@ export function ThreadAnnotationHoverPopover(props: { rowActive: boolean; trigger: ReactNode; threadDetails: ReactNode; + trailingContent?: ReactNode; onEdit: () => void; onResolve: () => void; onBodyChange: (body: string) => Promise<boolean>; @@ -379,6 +380,7 @@ export function ThreadAnnotationHoverPopover(props: { /> </div> </div> + {props.trailingContent} </div> </PopoverPopup> </Popover> diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index 569b4be96e62..829235b5daa6 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -4,7 +4,7 @@ import { scopeThreadRef, scopedThreadKey, } from "@t3tools/client-runtime/environment"; -import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { settlePromise } from "@t3tools/client-runtime/state/runtime"; import { canSettle, canSnooze, threadWokeAt } from "@t3tools/client-runtime/state/thread-settled"; import { EnvironmentId, type ScopedThreadRef, ThreadId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -17,7 +17,6 @@ import { getFallbackThreadIdAfterDelete, pinOrderKeyBetween } from "../component import { useComposerDraftStore } from "../composerDraftStore"; import { terminalEnvironment } from "../state/terminal"; import { threadEnvironment } from "../state/threads"; -import { vcsEnvironment } from "../state/vcs"; import { useNewThreadHandler } from "./useHandleNewThread"; import { refreshArchivedThreadsForEnvironment } from "../lib/archivedThreadsState"; import { readLocalApi } from "../localApi"; @@ -35,7 +34,6 @@ import { useTerminalUiStateStore } from "../terminalUiStateStore"; import { useUiStateStore } from "../uiStateStore"; import { buildThreadRouteParams, resolveThreadRouteRef } from "../threadRoutes"; import { formatWorktreePathForDisplay, getOrphanedWorktreePathForThread } from "../worktreeCleanup"; -import { stackedThreadToast, toastManager } from "../components/ui/toast"; import { useClientSettings } from "./useSettings"; import { useAtomCommand } from "../state/use-atom-command"; @@ -168,12 +166,6 @@ export function useThreadActions() { reportFailure: false, }); const stopThreadSession = useAtomCommand(threadEnvironment.stopSession); - const removeWorktree = useAtomCommand(vcsEnvironment.removeWorktree, { - reportFailure: false, - }); - const refreshVcsStatus = useAtomCommand(vcsEnvironment.refreshStatus, { - reportFailure: false, - }); const sidebarThreadSortOrder = useClientSettings((settings) => settings.sidebarThreadSortOrder); const confirmThreadDelete = useClientSettings((settings) => settings.confirmThreadDelete); const clearComposerDraftForThread = useComposerDraftStore((store) => store.clearDraftThread); @@ -358,7 +350,10 @@ export function useThreadActions() { }); const deleteResult = await deleteThreadMutation({ environmentId: threadRef.environmentId, - input: { threadId: threadRef.threadId }, + input: { + threadId: threadRef.threadId, + ...(shouldDeleteWorktree ? { deleteWorktree: true } : {}), + }, }); if (deleteResult._tag === "Failure") { return deleteResult; @@ -407,49 +402,6 @@ export function useThreadActions() { } } - if (!shouldDeleteWorktree || !orphanedWorktreePath || !threadProject) { - return deleteResult; - } - - const removeResult = await removeWorktree({ - environmentId: threadRef.environmentId, - input: { - cwd: threadProject.workspaceRoot, - path: orphanedWorktreePath, - force: true, - }, - }); - const refreshResult = - removeResult._tag === "Success" - ? await refreshVcsStatus({ - environmentId: threadRef.environmentId, - input: { cwd: threadProject.workspaceRoot }, - }) - : null; - const cleanupFailure = - removeResult._tag === "Failure" - ? removeResult - : refreshResult?._tag === "Failure" - ? refreshResult - : null; - if (cleanupFailure) { - const error = squashAtomCommandFailure(cleanupFailure); - const message = error instanceof Error ? error.message : "Unknown error removing worktree."; - console.error("Failed to remove orphaned worktree after thread deletion", { - threadId: threadRef.threadId, - projectCwd: threadProject.workspaceRoot, - worktreePath: orphanedWorktreePath, - error, - }); - toastManager.add( - stackedThreadToast({ - type: "error", - title: "Thread deleted, but worktree removal failed", - description: `Could not remove ${displayWorktreePath ?? orphanedWorktreePath}. ${message}`, - }), - ); - return cleanupFailure; - } return deleteResult; }, [ @@ -459,8 +411,6 @@ export function useThreadActions() { closeTerminal, deleteThreadMutation, getCurrentRouteThreadRef, - refreshVcsStatus, - removeWorktree, router, resolveThreadTarget, sidebarThreadSortOrder, diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md index cd64c48531a4..fc3b3761770d 100644 --- a/docs/lastcode/durable-worktree-cleanup-plan.md +++ b/docs/lastcode/durable-worktree-cleanup-plan.md @@ -56,6 +56,7 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - The decider chooses the initial deleting or queued state by inspecting unfinished cleanup jobs for the same repository. The resulting `thread.deleted` event contains the concrete cleanup record, making the user’s choice durable with the deletion. - Retry is valid only from failed. Abandonment is valid from deleting, queued, or failed. Internal lifecycle transitions validate their expected prior state. - A failed job is not an active queue blocker. +- Project deletion is rejected while any child thread still has a cleanup state, even with `force: true`; the user must wait for cleanup or choose **Keep worktree** first. ### Projection @@ -68,9 +69,9 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del ### Reactor - Extend `ThreadDeletionReactor`; it already owns provider-session and terminal cleanup for `thread.deleted`. -- Queue one scoped cleanup fiber per job behind a repository-keyed semaphore. Semaphore acquisition preserves same-repository serialization while different repositories proceed independently. -- Before a queued fiber runs, persist the started transition. Call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. -- Persist completed or failed with the exact structured error message. Refresh Git status after success as best effort; refresh failure must not turn a successful deletion into a failed cleanup. +- Queue jobs through one scoped drainable worker per repository. Each worker preserves same-repository order while workers for different repositories proceed independently. +- Before a queued job runs, persist the started transition. Call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. +- Persist completed or failed with the exact structured error message. - At startup, query and enqueue all resumable jobs after projections are bootstrapped. Tests wait on the reactor drain/receipts, never sleeps. ## Client presentation @@ -110,9 +111,9 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - Contract encode/decode tests for every cleanup state and command/event. - Decider tests for ownership, shared-worktree rejection, atomic initial state, retry, abandonment, and transition invariants. - Projection/migration/query tests for persistence, startup enumeration, shell inclusion, and final removal. -- Reactor tests with controlled removal effects proving same-repository serialization, cross-repository concurrency, restart recovery, success, failure, retry, and drain semantics. -- Client-runtime reducer tests proving tombstones survive upserts and disappear on completion/abandonment. -- Web logic/component tests for status priority, disabled/clickable semantics, hover order/content, failure dialog actions, and both sidebar variants. +- Reactor tests with controlled removal effects proving same-repository ordering, restart recovery, success, failure, queue continuation, and drain semantics. +- Projection and shell-stream tests proving tombstones survive upserts and disappear on completion/abandonment. +- Web logic tests for status priority, route fallback exclusion, and cleanup pills shared by both sidebar variants. - Mobile presentation tests proving observe-only status priority. - Focused lint and package typechecks, committed-range `git diff --check`, and the guarded LastCode quick-CI push gate. - Integrated disposable-state web QA in both sidebar versions, light and dark themes, covering deleting, queued, failure, Retry, Copy details, Keep worktree, navigation fallback, and silent success. Capture before/after images; record motion if elapsed/transition behavior needs review. @@ -125,4 +126,4 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - Providers: not provider-shaped; existing deletion reactor stops any provider session first. - Contracts/server/projections: required for durable remote and multi-device behavior. - Local, relay, and tunnel connections: use the same persisted orchestration commands and shell stream; disconnects do not own job lifetime. -- Documentation: this LastCode implementation plan records fork-only behavior; no upstream user documentation changes in this PR. +- Documentation: the implementation plan and LastCode section of the user sidebar guide document this fork-only behavior. diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index c90ef9db25c0..3409f8d62cb6 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -29,6 +29,18 @@ changed. Editing, resolving, or reopening the annotation moves that marker to th Resolved annotations disappear from the sidebar and composer but remain available from their yellow minimap marker, where they can be edited or reopened. +## Worktree cleanup in LastCode + +When you delete a thread and choose to delete its worktree, the thread stays in the sidebar until +the server finishes that cleanup. **Deleting** means removal is active. **Deleting (Queued)** means +another worktree from the same repository is being removed first; hover the row to see which +thread it is waiting for. Cleanup for different repositories can proceed at the same time. + +If cleanup fails, the row changes to **Cleanup failed**. Select anywhere on that row to see the +error and choose **Retry**, **Copy details**, or **Keep worktree**. LastCode resumes unfinished +cleanup after a server restart. Mobile shows these states but recovery actions are available only +on web and desktop. + ## Environment artwork Dev and Nightly environments can identify themselves with artwork at the top of the sidebar and in diff --git a/packages/client-runtime/src/operations/commands.ts b/packages/client-runtime/src/operations/commands.ts index bbbd3b873fa5..2b6f5bd8f939 100644 --- a/packages/client-runtime/src/operations/commands.ts +++ b/packages/client-runtime/src/operations/commands.ts @@ -33,6 +33,8 @@ export type UpdateProjectInput = CommandInput<"project.meta.update">; export type DeleteProjectInput = CommandInput<"project.delete">; export type CreateThreadInput = CommandInput<"thread.create">; export type DeleteThreadInput = CommandInput<"thread.delete">; +export type RetryThreadWorktreeCleanupInput = CommandInput<"thread.worktree-cleanup.retry">; +export type AbandonThreadWorktreeCleanupInput = CommandInput<"thread.worktree-cleanup.abandon">; export type ArchiveThreadInput = CommandInput<"thread.archive">; export type UnarchiveThreadInput = CommandInput<"thread.unarchive">; export type SettleThreadInput = CommandInput<"thread.settle">; @@ -143,6 +145,27 @@ export const deleteThread: (input: DeleteThreadInput) => CommandEffect = Effect. }); }); +export const retryThreadWorktreeCleanup: (input: RetryThreadWorktreeCleanupInput) => CommandEffect = + Effect.fn("EnvironmentCommands.retryThreadWorktreeCleanup")(function* (input) { + return yield* dispatch({ + ...input, + type: "thread.worktree-cleanup.retry", + commandId: yield* commandId(input), + }); + }); + +export const abandonThreadWorktreeCleanup: ( + input: AbandonThreadWorktreeCleanupInput, +) => CommandEffect = Effect.fn("EnvironmentCommands.abandonThreadWorktreeCleanup")( + function* (input) { + return yield* dispatch({ + ...input, + type: "thread.worktree-cleanup.abandon", + commandId: yield* commandId(input), + }); + }, +); + export const archiveThread: (input: ArchiveThreadInput) => CommandEffect = Effect.fn( "EnvironmentCommands.archiveThread", )(function* (input) { diff --git a/packages/client-runtime/src/state/threadCommands.ts b/packages/client-runtime/src/state/threadCommands.ts index 6346f863158b..18f23ce80429 100644 --- a/packages/client-runtime/src/state/threadCommands.ts +++ b/packages/client-runtime/src/state/threadCommands.ts @@ -11,6 +11,8 @@ import { type ArchiveThreadInput, type CreateThreadInput, type DeleteThreadInput, + type RetryThreadWorktreeCleanupInput, + type AbandonThreadWorktreeCleanupInput, type InterruptThreadTurnInput, type RespondToThreadApprovalInput, type RespondToThreadUserInputInput, @@ -34,6 +36,8 @@ import { archiveThread, createThread, deleteThread, + retryThreadWorktreeCleanup, + abandonThreadWorktreeCleanup, interruptThreadTurn, respondToThreadApproval, respondToThreadUserInput, @@ -61,6 +65,8 @@ export type { ArchiveThreadInput, CreateThreadInput, DeleteThreadInput, + RetryThreadWorktreeCleanupInput, + AbandonThreadWorktreeCleanupInput, InterruptThreadTurnInput, RespondToThreadApprovalInput, RespondToThreadUserInputInput, @@ -105,6 +111,18 @@ export function createThreadEnvironmentAtoms<R, E>( scheduler, concurrency, }), + retryWorktreeCleanup: createEnvironmentCommand(runtime, { + label: "environment-data:commands:thread:worktree-cleanup:retry", + execute: (input: RetryThreadWorktreeCleanupInput) => retryThreadWorktreeCleanup(input), + scheduler, + concurrency, + }), + abandonWorktreeCleanup: createEnvironmentCommand(runtime, { + label: "environment-data:commands:thread:worktree-cleanup:abandon", + execute: (input: AbandonThreadWorktreeCleanupInput) => abandonThreadWorktreeCleanup(input), + scheduler, + concurrency, + }), archive: createEnvironmentCommand(runtime, { label: "environment-data:commands:thread:archive", execute: (input: ArchiveThreadInput) => archiveThread(input), diff --git a/packages/contracts/src/orchestration.test.ts b/packages/contracts/src/orchestration.test.ts index c185fb7b3ee6..95ae07fac97e 100644 --- a/packages/contracts/src/orchestration.test.ts +++ b/packages/contracts/src/orchestration.test.ts @@ -72,6 +72,47 @@ it.effect("decodes a dispatch error after its bootstrap thread was deleted", () }), ); +it.effect("decodes durable worktree cleanup commands and events", () => + Effect.gen(function* () { + const command = yield* decodeOrchestrationCommand({ + type: "thread.delete", + commandId: "cmd-delete-worktree", + threadId: "thread-cleanup", + deleteWorktree: true, + }); + assert.strictEqual(command.type, "thread.delete"); + if (command.type === "thread.delete") assert.strictEqual(command.deleteWorktree, true); + + const event = yield* decodeOrchestrationEvent({ + sequence: 1, + eventId: "event-delete-worktree", + aggregateKind: "thread", + aggregateId: "thread-cleanup", + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: "cmd-delete-worktree", + causationEventId: null, + correlationId: "cmd-delete-worktree", + metadata: {}, + payload: { + threadId: "thread-cleanup", + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: "2026-08-23T00:00:00.000Z", + blockedByThreadId: "thread-blocker", + }, + }, + }); + assert.strictEqual(event.type, "thread.deleted"); + if (event.type === "thread.deleted") { + assert.strictEqual(event.payload.worktreeCleanup?.status, "queued"); + } + }), +); + it.effect("parses turn diff input when fromTurnCount <= toTurnCount", () => Effect.gen(function* () { const parsed = yield* decodeTurnDiffInput({ diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 0f478707db5f..737b6b3aea01 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -462,6 +462,33 @@ export const ThreadAnnotation = Schema.Struct({ }); export type ThreadAnnotation = typeof ThreadAnnotation.Type; +const ThreadWorktreeCleanupBase = { + repositoryRoot: TrimmedNonEmptyString, + worktreePath: TrimmedNonEmptyString, +} as const; + +export const ThreadWorktreeCleanup = Schema.Union([ + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("deleting"), + startedAt: IsoDateTime, + }), + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("queued"), + queuedAt: IsoDateTime, + blockedByThreadId: ThreadId, + }), + Schema.Struct({ + ...ThreadWorktreeCleanupBase, + status: Schema.Literal("failed"), + startedAt: IsoDateTime, + failedAt: IsoDateTime, + error: Schema.String, + }), +]); +export type ThreadWorktreeCleanup = typeof ThreadWorktreeCleanup.Type; + export const OrchestrationThread = Schema.Struct({ id: ThreadId, projectId: ProjectId, @@ -504,6 +531,7 @@ export const OrchestrationThread = Schema.Struct({ // Command decisions use this projected marker to anchor annotations without // hydrating message bodies and attachments for every thread. latestUserMessageId: Schema.optional(Schema.NullOr(MessageId)), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), deletedAt: Schema.NullOr(IsoDateTime), messages: Schema.Array(OrchestrationMessage), proposedPlans: Schema.Array(OrchestrationProposedPlan).pipe( @@ -563,6 +591,7 @@ export const OrchestrationThreadShell = Schema.Struct({ pinOrderKey: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), titleRegeneration: Schema.optional(Schema.NullOr(ThreadTitleRegeneration)), annotation: Schema.optional(Schema.NullOr(ThreadAnnotation)), + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), session: Schema.NullOr(OrchestrationSession), latestUserMessageAt: Schema.NullOr(IsoDateTime), hasPendingApprovals: Schema.Boolean, @@ -784,6 +813,19 @@ const ThreadDeleteCommand = Schema.Struct({ type: Schema.Literal("thread.delete"), commandId: CommandId, threadId: ThreadId, + deleteWorktree: Schema.optional(Schema.Boolean), +}); + +const ThreadWorktreeCleanupRetryCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.retry"), + commandId: CommandId, + threadId: ThreadId, +}); + +const ThreadWorktreeCleanupAbandonCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.abandon"), + commandId: CommandId, + threadId: ThreadId, }); const ThreadArchiveCommand = Schema.Struct({ @@ -1038,6 +1080,8 @@ const DispatchableClientOrchestrationCommand = Schema.Union([ ProjectDeleteCommand, ThreadCreateCommand, ThreadDeleteCommand, + ThreadWorktreeCleanupRetryCommand, + ThreadWorktreeCleanupAbandonCommand, ThreadArchiveCommand, ThreadUnarchiveCommand, ThreadSettleCommand, @@ -1069,6 +1113,8 @@ export const ClientOrchestrationCommand = Schema.Union([ ProjectDeleteCommand, ThreadCreateCommand, ThreadDeleteCommand, + ThreadWorktreeCleanupRetryCommand, + ThreadWorktreeCleanupAbandonCommand, ThreadArchiveCommand, ThreadUnarchiveCommand, ThreadSettleCommand, @@ -1193,6 +1239,13 @@ const ThreadTurnAssistantFinalizeCommand = Schema.Struct({ createdAt: IsoDateTime, }); +const ThreadWorktreeCleanupUpdateCommand = Schema.Struct({ + type: Schema.Literal("thread.worktree-cleanup.update"), + commandId: CommandId, + threadId: ThreadId, + cleanup: Schema.NullOr(ThreadWorktreeCleanup), +}); + const InternalOrchestrationCommand = Schema.Union([ ThreadSessionSetCommand, ThreadMessageAssistantDeltaCommand, @@ -1204,6 +1257,7 @@ const InternalOrchestrationCommand = Schema.Union([ ThreadTitleRegenerationCompleteCommand, ThreadTurnRequestResolveCommand, ThreadTurnAssistantFinalizeCommand, + ThreadWorktreeCleanupUpdateCommand, ]); export type InternalOrchestrationCommand = typeof InternalOrchestrationCommand.Type; @@ -1219,6 +1273,7 @@ export const OrchestrationEventType = Schema.Literals([ "project.deleted", "thread.created", "thread.deleted", + "thread.worktree-cleanup-updated", "thread.archived", "thread.unarchived", "thread.settled", @@ -1303,6 +1358,13 @@ export const ThreadCreatedPayload = Schema.Struct({ export const ThreadDeletedPayload = Schema.Struct({ threadId: ThreadId, deletedAt: IsoDateTime, + worktreeCleanup: Schema.optional(Schema.NullOr(ThreadWorktreeCleanup)), +}); + +export const ThreadWorktreeCleanupUpdatedPayload = Schema.Struct({ + threadId: ThreadId, + cleanup: Schema.NullOr(ThreadWorktreeCleanup), + updatedAt: IsoDateTime, }); export const ThreadArchivedPayload = Schema.Struct({ @@ -1560,6 +1622,11 @@ export const OrchestrationEvent = Schema.Union([ type: Schema.Literal("thread.deleted"), payload: ThreadDeletedPayload, }), + Schema.Struct({ + ...EventBaseFields, + type: Schema.Literal("thread.worktree-cleanup-updated"), + payload: ThreadWorktreeCleanupUpdatedPayload, + }), Schema.Struct({ ...EventBaseFields, type: Schema.Literal("thread.archived"), From b38b11c10c8e6256cf63b5bd1b107ad97743bc08 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Sun, 23 Aug 2026 18:39:54 -0700 Subject: [PATCH 03/22] fix(lastcode): fence durable worktree cleanup --- .../Layers/ThreadDeletionReactor.test.ts | 123 +++++++++++++++++- .../Layers/ThreadDeletionReactor.ts | 56 +++++++- .../src/orchestration/decider.delete.test.ts | 26 ++++ apps/server/src/orchestration/decider.ts | 44 ++++++- .../Layers/ProjectionRepositories.test.ts | 6 +- .../persistence/Layers/ProjectionThreads.ts | 26 ++++ .../persistence/Services/ProjectionThreads.ts | 11 ++ apps/web/src/components/LegacySidebar.tsx | 5 +- apps/web/src/components/Sidebar.tsx | 4 + .../lastcode/durable-worktree-cleanup-plan.md | 7 +- 10 files changed, 289 insertions(+), 19 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 7a1c21cefb7e..193eb051383e 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -1,13 +1,17 @@ import { + CommandId, + EventId, GitCommandError, ProjectId, ProviderInstanceId, ThreadId, type OrchestrationCommand, + type OrchestrationEvent, type ThreadWorktreeCleanup, } from "@t3tools/contracts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; @@ -96,6 +100,91 @@ function cleanupRow( } describe("durable worktree cleanup", () => { + effectIt.live("tears down the thread before removing its worktree", () => + Effect.gen(function* () { + const thread = cleanupRow( + "cleanup-event", + { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/worktrees/event", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const deletedEvent: Extract<OrchestrationEvent, { type: "thread.deleted" }> = { + sequence: 1, + eventId: EventId.make("event-thread-deleted"), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make("command-thread-deleted"), + causationEventId: null, + correlationId: CommandId.make("command-thread-deleted"), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; + const rows = new Map([[thread.threadId, thread]]); + const operations: string[] = []; + const removed = yield* Deferred.make<void>(); + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.make(deletedEvent), + latestSequence: Effect.succeed(1), + readEvents: () => Stream.empty, + dispatch: (command) => { + if (command.type === "thread.worktree-cleanup.update") { + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: 2 }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path }) => + Effect.sync(() => operations.push(`remove:${path}`)).pipe( + Effect.andThen(Deferred.succeed(removed, undefined)), + ), + }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => void operations.push(`stop:${threadId}`)), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => void operations.push(`close:${threadId}`)), + }), + NodeServices.layer, + ); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(dependencies), + Layer.merge(dependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Deferred.await(removed); + yield* reactor.drain; + }).pipe(Effect.provide(testLayer)); + + expect(operations).toEqual([ + `stop:${thread.threadId}`, + `close:${thread.threadId}`, + "remove:/worktrees/event", + ]); + }), + ); + effectIt.live("resumes same-repository cleanup in order and persists failures", () => Effect.gen(function* () { const root = "/repo"; @@ -131,12 +220,17 @@ describe("durable worktree cleanup", () => { }, "2026-08-23T00:00:02.000Z", ); + const activeOwner = { + threadId: ThreadId.make("active-owner"), + worktreePath: third.worktreePath ?? "/worktrees/third", + }; const rows = new Map([ [first.threadId, first], [second.threadId, second], [third.threadId, third], ]); const removals: string[] = []; + const operations: string[] = []; const updates: Array< Extract<OrchestrationCommand, { type: "thread.worktree-cleanup.update" }> > = []; @@ -158,11 +252,13 @@ describe("durable worktree cleanup", () => { Layer.mock(ProjectionThreadRepository)({ getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), listPendingWorktreeCleanup: () => Effect.succeed([first, second, third]), + listActiveWorktreeOwners: () => Effect.succeed([activeOwner]), }), Layer.mock(GitWorkflowService)({ removeWorktree: ({ path }) => Effect.gen(function* () { removals.push(path); + operations.push(`remove:${path}`); if (path === second.worktreePath) { return yield* new GitCommandError({ operation: "remove worktree", @@ -173,8 +269,13 @@ describe("durable worktree cleanup", () => { } }), }), - Layer.mock(ProviderService)({ stopSession: () => Effect.void }), - Layer.mock(TerminalManager.TerminalManager)({ close: () => Effect.void }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => void operations.push(`stop:${threadId}`)), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => void operations.push(`close:${threadId}`)), + }), NodeServices.layer, ); const testLayer = ThreadDeletionReactorLive.pipe( @@ -188,7 +289,17 @@ describe("durable worktree cleanup", () => { yield* reactor.drain; }).pipe(Effect.provide(testLayer)); - expect(removals).toEqual(["/worktrees/first", "/worktrees/second", "/worktrees/third"]); + expect(removals).toEqual(["/worktrees/first", "/worktrees/second"]); + expect(operations).toEqual([ + `stop:${first.threadId}`, + `close:${first.threadId}`, + "remove:/worktrees/first", + `stop:${second.threadId}`, + `close:${second.threadId}`, + "remove:/worktrees/second", + `stop:${third.threadId}`, + `close:${third.threadId}`, + ]); expect( updates.map((command) => [command.threadId, command.cleanup?.status ?? "complete"]), ).toEqual([ @@ -196,12 +307,16 @@ describe("durable worktree cleanup", () => { [second.threadId, "deleting"], [second.threadId, "failed"], [third.threadId, "deleting"], - [third.threadId, "complete"], + [third.threadId, "failed"], ]); expect(updates[2]?.cleanup).toMatchObject({ status: "failed", error: expect.stringContaining("permission denied"), }); + expect(updates[4]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("active-owner"), + }); }), ); }); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 1c0505e7857d..b3280d2e422d 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -1,5 +1,6 @@ import { CommandId, type OrchestrationEvent, type ThreadWorktreeCleanup } from "@t3tools/contracts"; import { makeDrainableWorker, type DrainableWorker } from "@t3tools/shared/DrainableWorker"; +import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; @@ -26,6 +27,7 @@ type PendingCleanup = Exclude<ThreadWorktreeCleanup, { readonly status: "failed" type CleanupJob = { readonly threadId: ThreadDeletedEvent["payload"]["threadId"]; readonly cleanup: PendingCleanup; + readonly needsTeardown: boolean; }; export const logCleanupCauseUnlessInterrupted = <R, E>({ @@ -116,6 +118,11 @@ const make = Effect.gen(function* () { }); const processCleanup = Effect.fn("processThreadWorktreeCleanup")(function* (job: CleanupJob) { + if (job.needsTeardown) { + yield* stopProviderSession(job.threadId); + yield* closeThreadTerminals(job.threadId); + } + const projected = yield* projectionThreads.getById({ threadId: job.threadId }); if (Option.isNone(projected)) return; const current = projected.value.worktreeCleanup; @@ -132,6 +139,23 @@ const make = Effect.gen(function* () { yield* dispatchCleanup(job.threadId, deleting); } + const normalizedWorktreePath = normalizeProjectPathForComparison(deleting.worktreePath); + const activeOwner = (yield* projectionThreads.listActiveWorktreeOwners()).find( + (candidate) => + candidate.threadId !== job.threadId && + normalizeProjectPathForComparison(candidate.worktreePath) === normalizedWorktreePath, + ); + if (activeOwner !== undefined) { + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: `Worktree '${deleting.worktreePath}' is now used by active thread '${activeOwner.threadId}'.`, + }); + return; + } + const removal = yield* Effect.result( gitWorkflow.removeWorktree({ cwd: deleting.repositoryRoot, @@ -140,7 +164,17 @@ const make = Effect.gen(function* () { }), ); if (Result.isSuccess(removal)) { - yield* dispatchCleanup(job.threadId, null); + yield* dispatchCleanup(job.threadId, null).pipe( + Effect.retry({ times: 2 }), + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); + return Effect.logWarning("removed worktree but could not persist cleanup completion", { + threadId: job.threadId, + worktreePath: deleting.worktreePath, + cause: Cause.pretty(cause), + }); + }), + ); return; } @@ -223,13 +257,21 @@ const make = Effect.gen(function* () { const cleanup = event.payload.worktreeCleanup; return cleanup == null || cleanup.status === "failed" ? Effect.void - : enqueueCleanup({ threadId: event.payload.threadId, cleanup }); + : enqueueCleanup({ + threadId: event.payload.threadId, + cleanup, + needsTeardown: false, + }); } if (event.type === "thread.worktree-cleanup-updated") { const cleanup = event.payload.cleanup; return cleanup == null || cleanup.status === "failed" ? Effect.void - : enqueueCleanup({ threadId: event.payload.threadId, cleanup }); + : enqueueCleanup({ + threadId: event.payload.threadId, + cleanup, + needsTeardown: false, + }); } return Effect.void; }; @@ -245,9 +287,9 @@ const make = Effect.gen(function* () { yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if (event.type === "thread.deleted") { - return Effect.all([worker.enqueue(event), enqueueCleanupFromEvent(event)]).pipe( - Effect.asVoid, - ); + return worker + .enqueue(event) + .pipe(Effect.andThen(worker.drain), Effect.andThen(enqueueCleanupFromEvent(event))); } return enqueueCleanupFromEvent(event); }), @@ -259,7 +301,7 @@ const make = Effect.gen(function* () { const cleanup = thread.worktreeCleanup; return cleanup == null || cleanup.status === "failed" ? Effect.void - : enqueueCleanup({ threadId: thread.threadId, cleanup }); + : enqueueCleanup({ threadId: thread.threadId, cleanup, needsTeardown: true }); }), ), Effect.catchCause((cause) => diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index 6235131bd92e..9be29c84092c 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -245,6 +245,32 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { readModel, })) as PlannedThreadDeletedEvent; const afterDelete = yield* projectEvent(readModel, { ...deleted, sequence: 4 }); + + const earlyAbandonError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.worktree-cleanup.abandon", + commandId: asCommandId("cmd-cleanup-abandon-early"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterDelete, + }), + ); + expect(earlyAbandonError.message).toContain("does not have failed worktree cleanup"); + + const pathReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.meta.update", + commandId: asCommandId("cmd-cleanup-path-reuse"), + threadId: asThreadId("thread-delete-2"), + worktreePath: "/tmp/project-delete-worktrees/cleanup-retry", + }, + readModel: afterDelete, + }), + ); + expect(pathReuseError.message).toContain("is still being cleaned up by thread"); + const failed = yield* decideOrchestrationCommand({ command: { type: "thread.worktree-cleanup.update", diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index c3f7c0fa1e53..5a9c9c6a6b3d 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -200,6 +200,20 @@ function findWorktreeCleanupBlocker( })[0]; } +function findWorktreeCleanupOwner( + readModel: OrchestrationReadModel, + worktreePath: string, + exceptThreadId?: string, +) { + const normalizedPath = normalizeProjectPathForComparison(worktreePath); + return readModel.threads.find( + (candidate) => + candidate.id !== exceptThreadId && + candidate.worktreeCleanup != null && + normalizeProjectPathForComparison(candidate.worktreeCleanup.worktreePath) === normalizedPath, + ); +} + function withEventBase( input: Pick<OrchestrationCommand, "commandId"> & { readonly aggregateKind: OrchestrationEvent["aggregateKind"]; @@ -424,6 +438,15 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" command, threadId: command.threadId, }); + if (command.worktreePath !== null) { + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.worktreePath); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${command.worktreePath}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } + } return { ...(yield* withEventBase({ aggregateKind: "thread", @@ -556,10 +579,14 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" case "thread.worktree-cleanup.abandon": { const thread = yield* requireThread({ readModel, command, threadId: command.threadId }); - if (thread.deletedAt === null || thread.worktreeCleanup == null) { + if ( + thread.deletedAt === null || + thread.worktreeCleanup == null || + thread.worktreeCleanup.status !== "failed" + ) { return yield* new OrchestrationCommandInvariantError({ commandType: command.type, - detail: `Thread '${thread.id}' does not have worktree cleanup to abandon.`, + detail: `Thread '${thread.id}' does not have failed worktree cleanup to abandon.`, }); } const occurredAt = yield* nowIso; @@ -1147,6 +1174,19 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" thread.branch !== command.expectedBranch ? thread.branch : command.branch; + if (command.worktreePath != null) { + const cleanupOwner = findWorktreeCleanupOwner( + readModel, + command.worktreePath, + command.threadId, + ); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${command.worktreePath}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } + } const occurredAt = yield* nowIso; return { ...(yield* withEventBase({ diff --git a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts index 128430c13eb0..a326e5367e68 100644 --- a/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts +++ b/apps/server/src/persistence/Layers/ProjectionRepositories.test.ts @@ -98,7 +98,7 @@ projectionRepositoriesLayer("Projection repositories", (it) => { runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: null, + worktreePath: "/tmp/thread-null-options-worktree", latestTurnId: null, createdAt: "2026-03-24T00:00:00.000Z", updatedAt: "2026-03-24T00:00:00.000Z", @@ -167,6 +167,10 @@ projectionRepositoriesLayer("Projection repositories", (it) => { Option.getOrNull(persisted)?.latestUserMessageId, MessageId.make("message-1"), ); + assert.deepStrictEqual( + (yield* threads.listActiveWorktreeOwners()).map((thread) => thread.threadId), + [ThreadId.make("thread-null-options")], + ); }), ); diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index 089f8ff01a48..6058acdfc152 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -7,8 +7,10 @@ import * as Struct from "effect/Struct"; import { toPersistenceSqlError } from "../Errors.ts"; import { + ActiveWorktreeOwner, DeleteProjectionThreadInput, GetProjectionThreadInput, + ListActiveWorktreeOwnerThreadsInput, ListProjectionThreadsByProjectInput, ListPendingWorktreeCleanupThreadsInput, ProjectionThread, @@ -254,6 +256,21 @@ const makeProjectionThreadRepository = Effect.gen(function* () { `, }); + const listActiveWorktreeOwnerRows = SqlSchema.findAll({ + Request: ListActiveWorktreeOwnerThreadsInput, + Result: ActiveWorktreeOwner, + execute: () => + sql` + SELECT + thread_id AS "threadId", + worktree_path AS "worktreePath" + FROM projection_threads + WHERE deleted_at IS NULL + AND worktree_path IS NOT NULL + ORDER BY created_at ASC, thread_id ASC + `, + }); + const upsert: ProjectionThreadRepositoryShape["upsert"] = (row) => upsertProjectionThreadRow(row).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.upsert:query")), @@ -277,6 +294,14 @@ const makeProjectionThreadRepository = Effect.gen(function* () { ), ); + const listActiveWorktreeOwners: ProjectionThreadRepositoryShape["listActiveWorktreeOwners"] = + () => + listActiveWorktreeOwnerRows(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionThreadRepository.listActiveWorktreeOwners:query"), + ), + ); + const deleteById: ProjectionThreadRepositoryShape["deleteById"] = (input) => deleteProjectionThreadRow(input).pipe( Effect.mapError(toPersistenceSqlError("ProjectionThreadRepository.deleteById:query")), @@ -287,6 +312,7 @@ const makeProjectionThreadRepository = Effect.gen(function* () { getById, listByProjectId, listPendingWorktreeCleanup, + listActiveWorktreeOwners, deleteById, } satisfies ProjectionThreadRepositoryShape; }); diff --git a/apps/server/src/persistence/Services/ProjectionThreads.ts b/apps/server/src/persistence/Services/ProjectionThreads.ts index a15792ef4bf8..5d3d9eed2d0f 100644 --- a/apps/server/src/persistence/Services/ProjectionThreads.ts +++ b/apps/server/src/persistence/Services/ProjectionThreads.ts @@ -73,6 +73,12 @@ export const ListProjectionThreadsByProjectInput = Schema.Struct({ projectId: ProjectId, }); export const ListPendingWorktreeCleanupThreadsInput = Schema.Void; +export const ListActiveWorktreeOwnerThreadsInput = Schema.Void; +export const ActiveWorktreeOwner = Schema.Struct({ + threadId: ThreadId, + worktreePath: Schema.String, +}); +export type ActiveWorktreeOwner = typeof ActiveWorktreeOwner.Type; export type ListProjectionThreadsByProjectInput = typeof ListProjectionThreadsByProjectInput.Type; /** @@ -107,6 +113,11 @@ export interface ProjectionThreadRepositoryShape { ProjectionRepositoryError >; + readonly listActiveWorktreeOwners: () => Effect.Effect< + ReadonlyArray<ActiveWorktreeOwner>, + ProjectionRepositoryError + >; + /** * Soft-delete a projected thread row by id. */ diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index f920c7b25916..785aef9b297a 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -2099,8 +2099,8 @@ const SidebarProjectItem = memo(function SidebarProjectItem(props: SidebarProjec if (!confirmed) return; } - const deletedThreadKeys = new Set(threadKeys); - for (const { threadRef } of selectedThreadEntries) { + const deletedThreadKeys = new Set<string>(); + for (const { threadKey, threadRef } of selectedThreadEntries) { const result = await deleteThread(threadRef, { deletedThreadKeys, }); @@ -2117,6 +2117,7 @@ const SidebarProjectItem = memo(function SidebarProjectItem(props: SidebarProjec } return; } + deletedThreadKeys.add(threadKey); } removeFromSelection(threadKeys); }, diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index 498d90cf6d4d..f2eca9d1c599 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -2096,6 +2096,10 @@ export default function Sidebar() { const snoozed: EnvironmentThreadShell[] = []; const settled: EnvironmentThreadShell[] = []; for (const thread of visible) { + if (thread.worktreeCleanup != null) { + active.push(thread); + continue; + } // Threads on servers without the settlement capability (old server, // or descriptor not loaded yet) never classify as settled: the user // could neither un-settle nor pin them, so auto-settling them would diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md index fc3b3761770d..56512ac1b7c8 100644 --- a/docs/lastcode/durable-worktree-cleanup-plan.md +++ b/docs/lastcode/durable-worktree-cleanup-plan.md @@ -30,7 +30,7 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - A failed cleanup releases its repository queue so the next queued cleanup can run. - Retry re-enters the repository queue. If another cleanup owns it, persist queued state and its blocker; otherwise persist deleting state and start immediately. - Success clears the cleanup state and removes the tombstone without a toast. -- **Keep worktree** is the only abandonment path. It clears the cleanup state and removes the tombstone without deleting the worktree. +- **Keep worktree** is the only abandonment path and is available after a cleanup failure, when no removal is in flight. It clears the cleanup state and removes the tombstone without deleting the worktree. ### Failure dialog @@ -54,8 +54,9 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - `thread.delete` derives and validates cleanup ownership from the command read model. - The decider chooses the initial deleting or queued state by inspecting unfinished cleanup jobs for the same repository. The resulting `thread.deleted` event contains the concrete cleanup record, making the user’s choice durable with the deletion. -- Retry is valid only from failed. Abandonment is valid from deleting, queued, or failed. Internal lifecycle transitions validate their expected prior state. +- Retry and abandonment are valid only from failed. Internal lifecycle transitions validate their expected prior state. - A failed job is not an active queue blocker. +- Thread creation and metadata updates cannot assign a worktree path reserved by an unfinished cleanup. The reactor also rechecks projected active owners immediately before physical removal. - Project deletion is rejected while any child thread still has a cleanup state, even with `force: true`; the user must wait for cleanup or choose **Keep worktree** first. ### Projection @@ -70,7 +71,7 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - Extend `ThreadDeletionReactor`; it already owns provider-session and terminal cleanup for `thread.deleted`. - Queue jobs through one scoped drainable worker per repository. Each worker preserves same-repository order while workers for different repositories proceed independently. -- Before a queued job runs, persist the started transition. Call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. +- Provider-session and terminal teardown completes before worktree removal begins. Before a queued job runs, persist the started transition, recheck active ownership, then call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. - Persist completed or failed with the exact structured error message. - At startup, query and enqueue all resumable jobs after projections are bootstrapped. Tests wait on the reactor drain/receipts, never sleeps. From 6f3ce0ac93e1d60678ec642e977a71c39f9af420 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Sun, 23 Aug 2026 18:51:40 -0700 Subject: [PATCH 04/22] fix(lastcode): keep cleanup tombstones inert --- .../features/threads/thread-list-items.tsx | 2 +- .../features/threads/thread-list-v2-items.tsx | 2 +- apps/web/src/components/LegacySidebar.tsx | 12 ++++---- apps/web/src/components/Sidebar.tsx | 7 +++-- .../WorktreeCleanupFailureDialog.tsx | 30 +++++++++++-------- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index 00b394afad8d..c8fe853d7687 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -741,7 +741,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { <ControlPillMenu actions={cleanupPending ? [] : menuActions} onPressAction={handleMenuAction} - shouldOpenOnLongPress + shouldOpenOnLongPress={!cleanupPending} > {rowContent(close)} </ControlPillMenu> diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 074b53e10f90..a109e8232359 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -1004,7 +1004,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { : cardMenuActions } onPressAction={handleMenuAction} - shouldOpenOnLongPress + shouldOpenOnLongPress={!cleanupPending} > {rowContent(close)} </ControlPillMenu> diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index 785aef9b297a..294ca7a688b5 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -1510,9 +1510,9 @@ const SidebarProjectItem = memo(function SidebarProjectItem(props: SidebarProjec visibleProjectThreads.map((thread) => resolveProjectThreadStatus(thread)), ); return { - orderedProjectThreadKeys: visibleProjectThreads.map((thread) => - scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id)), - ), + orderedProjectThreadKeys: visibleProjectThreads + .filter((thread) => thread.worktreeCleanup == null) + .map((thread) => scopedThreadKey(scopeThreadRef(thread.environmentId, thread.id))), projectStatus, visibleProjectThreads, }; @@ -2022,12 +2022,14 @@ const SidebarProjectItem = memo(function SidebarProjectItem(props: SidebarProjec if (!api) return; const threadKeys = [...useThreadSelectionStore.getState().selectedThreadKeys]; if (threadKeys.length === 0) return; - const count = threadKeys.length; const selectedThreadEntries = threadKeys.flatMap((threadKey) => { const threadRef = parseScopedThreadKey(threadKey); const thread = threadRef ? readThreadShell(threadRef) : null; - return threadRef && thread ? [{ threadKey, threadRef, thread }] : []; + if (!threadRef || !thread || thread.worktreeCleanup != null) return []; + return [{ threadKey, threadRef, thread }]; }); + const count = selectedThreadEntries.length; + if (count === 0) return; const hasRunningThread = selectedThreadEntries.some( ({ thread }) => thread.session?.status === "running" && thread.session.activeTurnId != null, ); diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx index f2eca9d1c599..42653b2d980b 100644 --- a/apps/web/src/components/Sidebar.tsx +++ b/apps/web/src/components/Sidebar.tsx @@ -2175,7 +2175,10 @@ export default function Sidebar() { const [activeSearchResultIndex, setActiveSearchResultIndex] = useState(0); const isSearchingThreads = threadSearchQuery.trim().length > 0; const searchableThreads = useMemo( - () => [...pinnedThreads, ...activeThreads, ...snoozedThreads, ...settledThreads], + () => + [...pinnedThreads, ...activeThreads, ...snoozedThreads, ...settledThreads].filter( + (thread) => thread.worktreeCleanup == null, + ), [activeThreads, pinnedThreads, settledThreads, snoozedThreads], ); const threadSearchResults = useMemo( @@ -2905,7 +2908,7 @@ export default function Sidebar() { // thread deletion elsewhere) and the menu labels must count only what // the actions will touch. const threadKeys = [...useThreadSelectionStore.getState().selectedThreadKeys].filter( - (threadKey) => threadByKeyRef.current.has(threadKey), + (threadKey) => threadByKeyRef.current.get(threadKey)?.worktreeCleanup == null, ); if (threadKeys.length === 0) return; const count = threadKeys.length; diff --git a/apps/web/src/components/WorktreeCleanupFailureDialog.tsx b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx index 8277d5386374..490cf39e3198 100644 --- a/apps/web/src/components/WorktreeCleanupFailureDialog.tsx +++ b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx @@ -1,6 +1,7 @@ import type { SidebarThreadSummary } from "../types"; import { threadEnvironment } from "../state/threads"; import { useAtomCommand } from "../state/use-atom-command"; +import { ensureLocalApi } from "../localApi"; import { Button } from "./ui/button"; import { Dialog, @@ -32,6 +33,22 @@ export function WorktreeCleanupFailureDialog(props: { "", cleanup.error, ].join("\n"); + const keepWorktree = async () => { + const confirmed = await ensureLocalApi().dialogs.confirm( + [ + "Keep this worktree?", + "LastCode will stop trying to remove it and dismiss this cleanup failure.", + "You can still remove the worktree manually later.", + ].join("\n"), + { variant: "destructive" }, + ); + if (!confirmed) return; + const result = await abandon({ + environmentId: props.thread.environmentId, + input: { threadId: props.thread.id }, + }); + if (result._tag === "Success") props.onOpenChange(false); + }; return ( <Dialog open={props.open} onOpenChange={props.onOpenChange}> @@ -56,18 +73,7 @@ export function WorktreeCleanupFailureDialog(props: { </pre> </DialogPanel> <DialogFooter className="sm:flex-wrap"> - <Button - type="button" - variant="destructive-outline" - onClick={() => { - void abandon({ - environmentId: props.thread.environmentId, - input: { threadId: props.thread.id }, - }).then((result) => { - if (result._tag === "Success") props.onOpenChange(false); - }); - }} - > + <Button type="button" variant="destructive-outline" onClick={() => void keepWorktree()}> Keep worktree </Button> <Button From 83783faed991cc83ce4e28a964c5a238b370e270 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Sun, 23 Aug 2026 19:09:06 -0700 Subject: [PATCH 05/22] fix(lastcode): harden cleanup version boundaries --- apps/server/src/environment/ServerEnvironment.test.ts | 1 + apps/server/src/environment/ServerEnvironment.ts | 1 + .../Layers/ProjectionSnapshotQuery.test.ts | 4 ++++ .../orchestration/Layers/ProjectionSnapshotQuery.ts | 2 +- .../orchestration/Layers/ThreadDeletionReactor.test.ts | 2 +- .../src/orchestration/Layers/ThreadDeletionReactor.ts | 7 ++++--- apps/web/src/hooks/useThreadActions.ts | 6 +++++- apps/web/src/state/entities.ts | 9 +++++++++ docs/lastcode/durable-worktree-cleanup-plan.md | 1 + packages/contracts/src/environment.test.ts | 10 ++++++++++ packages/contracts/src/environment.ts | 3 +++ 11 files changed, 40 insertions(+), 6 deletions(-) diff --git a/apps/server/src/environment/ServerEnvironment.test.ts b/apps/server/src/environment/ServerEnvironment.test.ts index 0b6c2a527995..7c22df14d41f 100644 --- a/apps/server/src/environment/ServerEnvironment.test.ts +++ b/apps/server/src/environment/ServerEnvironment.test.ts @@ -93,6 +93,7 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => { expect(second.capabilities.pullRequests).toBe(true); expect(second.capabilities.threadTitleRegeneration).toBe(true); expect(second.capabilities.threadAnnotations).toBe(true); + expect(second.capabilities.threadWorktreeCleanup).toBe(true); expect(second.capabilities.agentActivityPublishing).toBe(false); }), ); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 11dd4319beb2..d14b849f5cdb 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -153,6 +153,7 @@ export const make = Effect.gen(function* () { threadPinReorder: true, threadTitleRegeneration: true, threadAnnotations: true, + threadWorktreeCleanup: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), ...(serverSelfUpdate === "boot-service" ? { serverSelfUpdateProgress: true } : {}), }, diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 6ff5d6ea24ec..8217ed56776d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -1652,6 +1652,10 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { worktreePath: "/tmp/deleted-project-worktrees/thread-deleted", startedAt: "2026-04-05T00:00:05.000Z", }); + const cleanupDetail = yield* snapshotQuery.getThreadDetailById( + ThreadId.make("thread-deleted"), + ); + assert.equal(cleanupDetail._tag, "None"); }), ); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index b8b19bb285f4..b1eafe6a1cc3 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -2623,7 +2623,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { ), ]); - if (Option.isNone(threadRow)) { + if (Option.isNone(threadRow) || threadRow.value.deletedAt !== null) { return Option.none<OrchestrationThread>(); } diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 193eb051383e..bd39c79b3425 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -202,7 +202,7 @@ describe("durable worktree cleanup", () => { "cleanup-second", { status: "queued", - repositoryRoot: root, + repositoryRoot: `${root}/`, worktreePath: "/worktrees/second", queuedAt: "2026-08-23T00:00:01.000Z", blockedByThreadId: first.threadId, diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index b3280d2e422d..54d399a2b7e3 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -225,16 +225,17 @@ const make = Effect.gen(function* () { const getCleanupWorker = Effect.fn("getThreadWorktreeCleanupWorker")(function* ( repositoryRoot: string, ) { - const existing = (yield* Ref.get(cleanupWorkersRef)).get(repositoryRoot); + const repositoryKey = normalizeProjectPathForComparison(repositoryRoot); + const existing = (yield* Ref.get(cleanupWorkersRef)).get(repositoryKey); if (existing) return existing; const created = yield* makeDrainableWorker((job: CleanupJob) => processCleanupSafely(job).pipe(Effect.ensuring(removeEnqueuedCleanupThreadId(job.threadId))), ); return yield* Ref.modify(cleanupWorkersRef, (workers) => { - const current = workers.get(repositoryRoot); + const current = workers.get(repositoryKey); if (current) return [current, workers] as const; const next = new Map(workers); - next.set(repositoryRoot, created); + next.set(repositoryKey, created); return [created, next] as const; }); }); diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index 829235b5daa6..f050c2de9ac7 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -25,6 +25,7 @@ import { readEnvironmentSupportsPinReorder, readEnvironmentSupportsSettlement, readEnvironmentSupportsSnooze, + readEnvironmentSupportsWorktreeCleanup, readEnvironmentThreadRefs, readProject, readThreadShell, @@ -304,7 +305,10 @@ export function useThreadActions() { const displayWorktreePath = orphanedWorktreePath ? formatWorktreePathForDisplay(orphanedWorktreePath) : null; - const canDeleteWorktree = orphanedWorktreePath !== null && threadProject !== null; + const canDeleteWorktree = + orphanedWorktreePath !== null && + threadProject !== null && + readEnvironmentSupportsWorktreeCleanup(threadRef.environmentId); const localApi = readLocalApi(); let shouldDeleteWorktree = false; if (canDeleteWorktree && localApi) { diff --git a/apps/web/src/state/entities.ts b/apps/web/src/state/entities.ts index 72e37a3c6a23..6b213711d512 100644 --- a/apps/web/src/state/entities.ts +++ b/apps/web/src/state/entities.ts @@ -277,6 +277,15 @@ export function readEnvironmentSupportsThreadAnnotations(environmentId: Environm ); } +/** Whether the server durably owns worktree cleanup after thread deletion. + Missing is unsupported so an older server cannot discard the user's cleanup choice. */ +export function readEnvironmentSupportsWorktreeCleanup(environmentId: EnvironmentId): boolean { + return ( + appAtomRegistry.get(environmentServerConfigsAtom).get(environmentId)?.environment.capabilities + .threadWorktreeCleanup === true + ); +} + export function readThreadDetail(ref: ScopedThreadRef): EnvironmentThread | null { return appAtomRegistry.get(environmentThreadDetails.detailAtom(ref)); } diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md index 56512ac1b7c8..23be2a537e30 100644 --- a/docs/lastcode/durable-worktree-cleanup-plan.md +++ b/docs/lastcode/durable-worktree-cleanup-plan.md @@ -12,6 +12,7 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - Keep the existing two-step delete confirmation. - Web and desktop may request worktree cleanup. Mobile may delete a thread but does not offer worktree deletion. +- Web and desktop offer the worktree option only when the connected server advertises durable worktree cleanup support; older servers receive a plain thread deletion instead of silently discarding the cleanup choice. - The server derives the repository root and worktree path from the authoritative project and thread records. The client sends only `deleteWorktree: true`; it does not supply paths. - Reject a cleanup request unless the thread owns a linked worktree that no other live thread uses. - Once deletion is accepted, immediately navigate an active thread route to the existing fallback. The tombstone is a sidebar status item, not an openable chat. diff --git a/packages/contracts/src/environment.test.ts b/packages/contracts/src/environment.test.ts index 35f94443f35d..3f745620a983 100644 --- a/packages/contracts/src/environment.test.ts +++ b/packages/contracts/src/environment.test.ts @@ -36,4 +36,14 @@ describe("ExecutionEnvironmentDescriptor", () => { }).capabilities.threadAnnotations, ).toBe(true); }); + + it("treats missing worktree cleanup as unsupported and preserves support", () => { + expect(decodeDescriptor(descriptor).capabilities.threadWorktreeCleanup).toBeUndefined(); + expect( + decodeDescriptor({ + ...descriptor, + capabilities: { ...descriptor.capabilities, threadWorktreeCleanup: true }, + }).capabilities.threadWorktreeCleanup, + ).toBe(true); + }); }); diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 2246d71a78ef..de424f6c89b5 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -70,6 +70,9 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ /** Server understands thread annotation create/edit/resolve/reopen commands and exposes annotation state in thread shell/detail snapshots. */ threadAnnotations: Schema.optionalKey(Schema.Boolean), + /** Server durably owns thread worktree cleanup after deletion and understands + deleteWorktree plus cleanup retry/abandon commands. */ + threadWorktreeCleanup: Schema.optionalKey(Schema.Boolean), /** The update path clients should offer for this server. Absent on servers that must be relaunched manually (dev checkouts, Windows foreground runs, pre-update servers). */ From d28d55fe0cfaa1ee7b8a881f13c1ccaf686498e3 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 00:58:15 -0700 Subject: [PATCH 06/22] fix(web): keep cleanup hover details readable --- apps/web/src/components/LegacySidebar.tsx | 27 ++++++++++++++++++- .../sidebar/SidebarThreadHoverContent.tsx | 11 ++++++-- .../lastcode/durable-worktree-cleanup-plan.md | 1 + 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/LegacySidebar.tsx b/apps/web/src/components/LegacySidebar.tsx index 294ca7a688b5..55bbcea60be5 100644 --- a/apps/web/src/components/LegacySidebar.tsx +++ b/apps/web/src/components/LegacySidebar.tsx @@ -548,7 +548,11 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr /> ); const cleanupHoverDetails = ( - <SidebarThreadCleanupHoverContent thread={thread} blockerTitle={cleanupBlockerTitle} /> + <SidebarThreadCleanupHoverContent + thread={thread} + blockerTitle={cleanupBlockerTitle} + standalone + /> ); const threadMetaClassName = isConfirmingArchive ? "pointer-events-none opacity-0" @@ -821,6 +825,27 @@ export const SidebarThreadRow = memo(function SidebarThreadRow(props: SidebarThr onKeyDown={handleRowKeyDown} onContextMenu={handleRowContextMenu} > + {isCleanupPending && !hasActiveAnnotation ? ( + <Tooltip> + <TooltipTrigger + render={ + <span + aria-label={`${thread.title} cleanup details`} + className="absolute inset-0 z-20 cursor-not-allowed" + /> + } + /> + <TooltipPopup + align="start" + className="max-w-80 text-left whitespace-normal [&_[data-slot=tooltip-viewport]]:p-0" + side="right" + sideOffset={4} + variant="glass" + > + {threadHoverDetails} + </TooltipPopup> + </Tooltip> + ) : null} <div className="flex min-w-0 flex-1 items-center gap-1.5 text-left"> {cleanup === null && prStatus && ( <Tooltip> diff --git a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx index d31f66beee11..0cad74c5cf09 100644 --- a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx +++ b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx @@ -124,16 +124,23 @@ export function SidebarThreadHoverContent(props: SidebarThreadHoverContentProps) export function SidebarThreadCleanupHoverContent(props: { thread: SidebarThreadSummary; blockerTitle: string | null; + standalone?: boolean; }) { const cleanup = props.thread.worktreeCleanup; if (cleanup == null || cleanup.status === "failed") return null; return ( - <div className="-mx-[var(--floating-content-inset)] -mb-[var(--floating-content-inset)] border-t border-orange-600/25 bg-orange-400 px-[var(--floating-content-inset)] py-2 text-xs text-foreground dark:bg-orange-400 dark:text-background"> + <div + className={cn( + !props.standalone && + "-mx-[var(--floating-content-inset)] -mb-[var(--floating-content-inset)]", + "border-t border-orange-600/25 bg-orange-400 px-[var(--floating-content-inset)] py-2 text-xs text-foreground dark:bg-orange-400 dark:text-background", + )} + > {cleanup.status === "deleting" ? ( <> <div className="font-medium">Deleting worktree</div> - <div className="mt-1 break-all font-mono text-[11px] opacity-80"> + <div className="mt-1 break-all font-mono text-[10px] text-wrap opacity-80"> {cleanup.worktreePath} </div> </> diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md index 23be2a537e30..8726e012d50e 100644 --- a/docs/lastcode/durable-worktree-cleanup-plan.md +++ b/docs/lastcode/durable-worktree-cleanup-plan.md @@ -50,6 +50,7 @@ The deleted thread remains visible as a temporary tombstone while cleanup is del - failed: `startedAt`, `failedAt`, `error` - Add client commands for retry and abandonment, plus internal commands/events for queued, started, failed, and completed transitions. - Keep new fields optional on the wire where compatibility with cached snapshots or older clients is required; absent means no cleanup. +- Pre-feature clients connected to a newer server are outside the compatibility boundary for tombstone presentation. Do not add a server-side presentation shim; clients and server should be upgraded together for this feature. ### Decision rules From 5b6f29f22e2191cecb1de780c462dab55dbd46c7 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 08:47:24 -0700 Subject: [PATCH 07/22] fix(lastcode): complete cleanup recovery paths --- .../features/threads/thread-list-v2-items.tsx | 94 ++++++++++--- .../src/features/threads/threadListV2.test.ts | 36 +++++ .../src/features/threads/threadListV2.ts | 9 ++ .../Layers/ThreadDeletionReactor.test.ts | 37 ++++- .../Layers/ThreadDeletionReactor.ts | 10 +- .../lastcode/durable-worktree-cleanup-plan.md | 132 ------------------ docs/user/thread-sidebar.md | 4 +- 7 files changed, 167 insertions(+), 155 deletions(-) delete mode 100644 docs/lastcode/durable-worktree-cleanup-plan.md diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index a109e8232359..126ff341db30 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -24,6 +24,7 @@ import { relativeTime } from "../../lib/time"; import { useThemeColor } from "../../lib/useThemeColor"; import type { PendingNewTask } from "../../state/use-pending-new-tasks"; import { terminalEnvironment } from "../../state/terminal"; +import { threadEnvironment } from "../../state/threads"; import { useAtomCommand } from "../../state/use-atom-command"; import { useThreadPr } from "../../state/use-thread-pr"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; @@ -34,6 +35,7 @@ import { resolveThreadListV2SnoozeGateExpiryMs, resolveThreadListV2Status, resolveThreadListV2SwipeActions, + resolveThreadListV2CleanupActions, type ThreadListV2Status, } from "./threadListV2"; import { ThreadSearchMatchExcerpt } from "./thread-search-match"; @@ -91,6 +93,11 @@ const LEGACY_MENU_ACTIONS: MenuAction[] = [ { id: "delete", title: "Delete", image: "trash", attributes: { destructive: true } }, ]; +const FAILED_CLEANUP_MENU_ACTIONS: MenuAction[] = [ + { id: "retry-worktree-cleanup", title: "Retry", image: "arrow.clockwise" }, + { id: "keep-worktree", title: "Keep worktree", image: "externaldrive" }, +]; + /** Rounded-row radius shared with the v1 sidebar rows. */ const SIDEBAR_V2_ROW_RADIUS = 12; @@ -406,8 +413,15 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const snoozedRow = props.snoozed === true; const pinnedRow = props.pinned === true; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; - const cleanupPending = thread.worktreeCleanup != null; + const cleanupFailed = resolveThreadListV2CleanupActions(thread.worktreeCleanup).length > 0; + const cleanupPending = thread.worktreeCleanup != null && !cleanupFailed; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); + const retryWorktreeCleanup = useAtomCommand(threadEnvironment.retryWorktreeCleanup, { + reportFailure: false, + }); + const abandonWorktreeCleanup = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { + reportFailure: false, + }); const pr = useThreadPr(thread, props.projectCwd ?? props.project?.workspaceRoot ?? null); const prState = pr?.state ?? null; @@ -469,6 +483,48 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { ); } }, [closeTerminal, runningAction, thread.environmentId, thread.id]); + const handleRetryWorktreeCleanup = useCallback(async () => { + const result = await retryWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }); + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not retry worktree cleanup", + error instanceof Error ? error.message : "The worktree cleanup could not be retried.", + ); + } + }, [retryWorktreeCleanup, thread.environmentId, thread.id]); + const handleKeepWorktree = useCallback(() => { + Alert.alert( + "Keep worktree?", + "LastCode will stop trying to remove this worktree. You can remove it manually later.", + [ + { text: "Cancel", style: "cancel" }, + { + text: "Keep worktree", + style: "destructive", + onPress: () => { + void abandonWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }).then((result) => { + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not keep worktree", + error instanceof Error + ? error.message + : "The worktree cleanup could not be dismissed.", + ); + } + }); + }, + }, + ], + ); + }, [abandonWorktreeCleanup, thread.environmentId, thread.id]); // Swipe: the v2 primary action is the lifecycle transition. Every settled // row can un-settle — explicit settles clear the override, auto-settled @@ -629,6 +685,8 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { if (nativeEvent.event === "archive") handleArchive(); if (nativeEvent.event === "regenerate-title") handleRegenerateTitle(); if (nativeEvent.event === "cancel-action") void handleCancelAction(); + if (nativeEvent.event === "retry-worktree-cleanup") void handleRetryWorktreeCleanup(); + if (nativeEvent.event === "keep-worktree") handleKeepWorktree(); if (nativeEvent.event === "delete") handleDelete(); const snoozeSelection = resolveThreadListV2SnoozeMenuSelection({ event: nativeEvent.event, @@ -645,7 +703,9 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { handleArchive, handleCancelAction, handleDelete, + handleKeepWorktree, handleRegenerateTitle, + handleRetryWorktreeCleanup, handleMovePinnedDown, handleMovePinnedUp, handlePin, @@ -853,7 +913,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { disabled={cleanupPending} onPress={() => { close(); - onSelectThread(thread); + if (!cleanupFailed) onSelectThread(thread); }} style={ sidebarPane @@ -895,7 +955,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { className={sidebarPane ? undefined : "bg-screen"} onPress={() => { close(); - onSelectThread(thread); + if (!cleanupFailed) onSelectThread(thread); }} style={ sidebarPane @@ -974,7 +1034,7 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { sidebarPane ? { borderRadius: SIDEBAR_V2_ROW_RADIUS, overflow: "hidden" } : undefined } enableTrackpadSwipe - enabled={!cleanupPending} + enabled={!cleanupPending && !cleanupFailed} // Full swipe commits the advertised lifecycle action (Settle / // Un-settle), never the secondary snooze action. fullSwipeAction="primary" @@ -991,20 +1051,22 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { {(close) => ( <ControlPillMenu actions={ - cleanupPending - ? [] - : snoozedRow - ? snoozedMenuActions - : !props.settlementSupported - ? legacyMenuActions - : canUnsettle - ? slimMenuActions - : swipeActions.secondary === "snooze" - ? snoozableCardMenuActions - : cardMenuActions + cleanupFailed + ? FAILED_CLEANUP_MENU_ACTIONS + : cleanupPending + ? [] + : snoozedRow + ? snoozedMenuActions + : !props.settlementSupported + ? legacyMenuActions + : canUnsettle + ? slimMenuActions + : swipeActions.secondary === "snooze" + ? snoozableCardMenuActions + : cardMenuActions } onPressAction={handleMenuAction} - shouldOpenOnLongPress={!cleanupPending} + shouldOpenOnLongPress={cleanupFailed || !cleanupPending} > {rowContent(close)} </ControlPillMenu> diff --git a/apps/mobile/src/features/threads/threadListV2.test.ts b/apps/mobile/src/features/threads/threadListV2.test.ts index c72cc159d6c9..0edfdc65afb6 100644 --- a/apps/mobile/src/features/threads/threadListV2.test.ts +++ b/apps/mobile/src/features/threads/threadListV2.test.ts @@ -17,6 +17,7 @@ import { buildThreadListV2Items, buildThreadListV2ListItems, resolveThreadListV2Enabled, + resolveThreadListV2CleanupActions, resolveThreadListV2SnoozeMenuSelection, resolveThreadListV2SnoozeGateExpiryMs, resolveThreadListV2Status, @@ -200,6 +201,41 @@ describe("resolveThreadListV2Status", () => { }); }); +describe("resolveThreadListV2CleanupActions", () => { + it("keeps failed cleanup tombstones recoverable from the mobile menu", () => { + expect( + resolveThreadListV2CleanupActions({ + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }), + ).toEqual(["retry-worktree-cleanup", "keep-worktree"]); + }); + + it("keeps queued and active cleanup tombstones menu-inert", () => { + expect( + resolveThreadListV2CleanupActions({ + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("blocking"), + }), + ).toEqual([]); + expect( + resolveThreadListV2CleanupActions({ + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + startedAt: NOW, + }), + ).toEqual([]); + }); +}); + describe("resolveThreadListV2SwipeActions", () => { it("offers settle and snooze for an active snoozable thread", () => { expect( diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index d0573d997242..eb2e77997a22 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -30,6 +30,15 @@ export { snoozeWakeLabel }; export type ThreadListV2Status = "approval" | "input" | "working" | "waiting" | "failed" | "ready"; export type ThreadListV2SwipeAction = "archive" | "settle" | "unsettle" | "snooze" | "unsnooze"; +export type ThreadListV2CleanupAction = "retry-worktree-cleanup" | "keep-worktree"; + +/** Failed cleanup tombstones stay reachable on mobile through recovery actions. */ +export function resolveThreadListV2CleanupActions( + cleanup: EnvironmentThreadShell["worktreeCleanup"], +): readonly ThreadListV2CleanupAction[] { + return cleanup?.status === "failed" ? ["retry-worktree-cleanup", "keep-worktree"] : []; +} + export function resolveThreadListV2SnoozeMenuSelection(input: { readonly event: string; readonly displayedPresets: ReadonlyArray<SnoozePreset>; diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index bd39c79b3425..7a01b988e0bb 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -188,6 +188,7 @@ describe("durable worktree cleanup", () => { effectIt.live("resumes same-repository cleanup in order and persists failures", () => Effect.gen(function* () { const root = "/repo"; + const existingWorktreePath = process.cwd(); const first = cleanupRow( "cleanup-first", { @@ -203,7 +204,7 @@ describe("durable worktree cleanup", () => { { status: "queued", repositoryRoot: `${root}/`, - worktreePath: "/worktrees/second", + worktreePath: existingWorktreePath, queuedAt: "2026-08-23T00:00:01.000Z", blockedByThreadId: first.threadId, }, @@ -220,6 +221,16 @@ describe("durable worktree cleanup", () => { }, "2026-08-23T00:00:02.000Z", ); + const fourth = cleanupRow( + "cleanup-already-removed", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/already-removed", + startedAt: "2026-08-23T00:00:03.000Z", + }, + "2026-08-23T00:00:03.000Z", + ); const activeOwner = { threadId: ThreadId.make("active-owner"), worktreePath: third.worktreePath ?? "/worktrees/third", @@ -228,6 +239,7 @@ describe("durable worktree cleanup", () => { [first.threadId, first], [second.threadId, second], [third.threadId, third], + [fourth.threadId, fourth], ]); const removals: string[] = []; const operations: string[] = []; @@ -251,7 +263,7 @@ describe("durable worktree cleanup", () => { }), Layer.mock(ProjectionThreadRepository)({ getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), - listPendingWorktreeCleanup: () => Effect.succeed([first, second, third]), + listPendingWorktreeCleanup: () => Effect.succeed([first, second, third, fourth]), listActiveWorktreeOwners: () => Effect.succeed([activeOwner]), }), Layer.mock(GitWorkflowService)({ @@ -267,6 +279,14 @@ describe("durable worktree cleanup", () => { detail: "permission denied", }); } + if (path === fourth.worktreePath) { + return yield* new GitCommandError({ + operation: "remove worktree", + command: "git worktree remove", + cwd: root, + detail: "not a working tree", + }); + } }), }), Layer.mock(ProviderService)({ @@ -289,16 +309,23 @@ describe("durable worktree cleanup", () => { yield* reactor.drain; }).pipe(Effect.provide(testLayer)); - expect(removals).toEqual(["/worktrees/first", "/worktrees/second"]); + expect(removals).toEqual([ + "/worktrees/first", + second.worktreePath, + "/worktrees/already-removed", + ]); expect(operations).toEqual([ `stop:${first.threadId}`, `close:${first.threadId}`, "remove:/worktrees/first", `stop:${second.threadId}`, `close:${second.threadId}`, - "remove:/worktrees/second", + `remove:${second.worktreePath}`, `stop:${third.threadId}`, `close:${third.threadId}`, + `stop:${fourth.threadId}`, + `close:${fourth.threadId}`, + "remove:/worktrees/already-removed", ]); expect( updates.map((command) => [command.threadId, command.cleanup?.status ?? "complete"]), @@ -308,6 +335,7 @@ describe("durable worktree cleanup", () => { [second.threadId, "failed"], [third.threadId, "deleting"], [third.threadId, "failed"], + [fourth.threadId, "complete"], ]); expect(updates[2]?.cleanup).toMatchObject({ status: "failed", @@ -317,6 +345,7 @@ describe("durable worktree cleanup", () => { status: "failed", error: expect.stringContaining("active-owner"), }); + expect(updates[5]?.cleanup).toBeNull(); }), ); }); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 54d399a2b7e3..4004f4db2f42 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -5,6 +5,7 @@ import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; @@ -57,6 +58,7 @@ const make = Effect.gen(function* () { const projectionThreads = yield* ProjectionThreadRepository; const providerService = yield* ProviderService; const terminalManager = yield* TerminalManager.TerminalManager; + const fileSystem = yield* FileSystem.FileSystem; const crypto = yield* Crypto.Crypto; const cleanupWorkersRef = yield* Ref.make<ReadonlyMap<string, DrainableWorker<CleanupJob>>>( new Map(), @@ -163,7 +165,13 @@ const make = Effect.gen(function* () { force: true, }), ); - if (Result.isSuccess(removal)) { + // A restart can observe `deleting` after Git removed the worktree but + // before the completion event was persisted. Git quite reasonably rejects + // a second removal of an unregistered path, so an absent path is already + // the desired end state and should complete the durable cleanup. + const alreadyRemoved = + Result.isFailure(removal) && !(yield* fileSystem.exists(deleting.worktreePath)); + if (Result.isSuccess(removal) || alreadyRemoved) { yield* dispatchCleanup(job.threadId, null).pipe( Effect.retry({ times: 2 }), Effect.catchCause((cause) => { diff --git a/docs/lastcode/durable-worktree-cleanup-plan.md b/docs/lastcode/durable-worktree-cleanup-plan.md deleted file mode 100644 index 8726e012d50e..000000000000 --- a/docs/lastcode/durable-worktree-cleanup-plan.md +++ /dev/null @@ -1,132 +0,0 @@ -# Durable Thread Worktree Cleanup Plan - -## Goal - -Deleting a thread with “Delete the worktree too” must create a durable cleanup job in the same persisted domain event as the thread deletion. The server owns that job until it succeeds or the user explicitly chooses **Keep worktree**. Closing a client, losing the connection, or restarting the server must not lose the cleanup. - -The deleted thread remains visible as a temporary tombstone while cleanup is deleting, queued, or failed. Successful cleanup and explicit abandonment remove the tombstone silently. - -## Product behavior - -### Confirmation and navigation - -- Keep the existing two-step delete confirmation. -- Web and desktop may request worktree cleanup. Mobile may delete a thread but does not offer worktree deletion. -- Web and desktop offer the worktree option only when the connected server advertises durable worktree cleanup support; older servers receive a plain thread deletion instead of silently discarding the cleanup choice. -- The server derives the repository root and worktree path from the authoritative project and thread records. The client sends only `deleteWorktree: true`; it does not supply paths. -- Reject a cleanup request unless the thread owns a linked worktree that no other live thread uses. -- Once deletion is accepted, immediately navigate an active thread route to the existing fallback. The tombstone is a sidebar status item, not an openable chat. -- Archive never schedules worktree cleanup. - -### Lifecycle and queueing - -- Persist cleanup intent and its initial state atomically in `thread.deleted`. -- Cleanup states are a separate lifecycle axis from agent states such as Working, Waiting, approvals, and input requests. -- The visible states are: - - `deleting`: orange **Deleting**; the v2 sidebar also shows elapsed time. - - `queued`: orange **Deleting (Queued)** with the blocking thread ID and title in hover details. - - `failed`: red **Cleanup failed**; any click on the row opens the failure dialog. -- Cleanups for the same repository run one at a time in deletion order. Cleanups for unrelated repositories may run concurrently. -- On server start, reload deleting and queued jobs from the projection and resume them in deletion order. Treat a stale deleting state as resumable work. -- A failed cleanup releases its repository queue so the next queued cleanup can run. -- Retry re-enters the repository queue. If another cleanup owns it, persist queued state and its blocker; otherwise persist deleting state and start immediately. -- Success clears the cleanup state and removes the tombstone without a toast. -- **Keep worktree** is the only abandonment path and is available after a cleanup failure, when no removal is in flight. It clears the cleanup state and removes the tombstone without deleting the worktree. - -### Failure dialog - -- Clicking anywhere on a failed row opens a dialog containing the thread identity, worktree path, and exact cleanup error. -- Actions are **Retry**, **Copy details**, and destructive-looking but non-destructive **Keep worktree**. -- Retry closes the dialog after the server accepts it. Copy details keeps it open. Keep worktree requires confirmation because it permanently stops automatic cleanup. - -## Domain and persistence design - -### Contracts - -- Extend `thread.delete` with optional `deleteWorktree: boolean`. -- Add a `ThreadWorktreeCleanup` discriminated union to thread shell/detail contracts. Each variant carries the authoritative repository root and worktree path: - - deleting: `startedAt` - - queued: `queuedAt`, `blockedByThreadId` - - failed: `startedAt`, `failedAt`, `error` -- Add client commands for retry and abandonment, plus internal commands/events for queued, started, failed, and completed transitions. -- Keep new fields optional on the wire where compatibility with cached snapshots or older clients is required; absent means no cleanup. -- Pre-feature clients connected to a newer server are outside the compatibility boundary for tombstone presentation. Do not add a server-side presentation shim; clients and server should be upgraded together for this feature. - -### Decision rules - -- `thread.delete` derives and validates cleanup ownership from the command read model. -- The decider chooses the initial deleting or queued state by inspecting unfinished cleanup jobs for the same repository. The resulting `thread.deleted` event contains the concrete cleanup record, making the user’s choice durable with the deletion. -- Retry and abandonment are valid only from failed. Internal lifecycle transitions validate their expected prior state. -- A failed job is not an active queue blocker. -- Thread creation and metadata updates cannot assign a worktree path reserved by an unfinished cleanup. The reactor also rechecks projected active owners immediately before physical removal. -- Project deletion is rejected while any child thread still has a cleanup state, even with `force: true`; the user must wait for cleanup or choose **Keep worktree** first. - -### Projection - -- Add a nullable JSON cleanup column to `projection_threads` through the next migration. -- Project every cleanup transition into that column. -- Include soft-deleted rows in shell snapshots and per-thread shell lookups only while cleanup is non-null. Completed or abandoned jobs disappear through the existing `thread-removed` shell event. -- Make `thread.deleted` use the same projection-backed upsert-or-remove decision as other thread events so the initial tombstone reaches every connected client. -- Expose a repository query for resumable deleting/queued jobs ordered by deletion time and thread ID. - -### Reactor - -- Extend `ThreadDeletionReactor`; it already owns provider-session and terminal cleanup for `thread.deleted`. -- Queue jobs through one scoped drainable worker per repository. Each worker preserves same-repository order while workers for different repositories proceed independently. -- Provider-session and terminal teardown completes before worktree removal begins. Before a queued job runs, persist the started transition, recheck active ownership, then call the server `GitWorkflowService.removeWorktree` primitive from PR #74 with `force: true`. -- Persist completed or failed with the exact structured error message. -- At startup, query and enqueue all resumable jobs after projections are bootstrapped. Tests wait on the reactor drain/receipts, never sleeps. - -## Client presentation - -### Shared behavior - -- Cleanup state overrides agent status once deletion is accepted. -- Deleting and queued rows are muted, non-selectable, and use a no-entry cursor. Failed rows are clickable only to open their dialog. -- Preserve the thread title, short ID, project grouping, branch, and muted `FolderGit2Icon`. Do not invent a new worktree glyph or color the worktree icon orange. -- Exclude tombstones from keyboard thread traversal, bulk thread actions, drag/reorder, route fallback candidates, and unread/settled calculations. - -### Legacy sidebar - -- Render orange `• Deleting` and `• Deleting (Queued)` labels with the existing status-label proportions. -- Render red `• Cleanup failed`; the whole row is the dialog target. -- Append a cleanup segment after the regular hover content and after any annotation: - - deleting: `Deleting worktree` and the formatted path. - - queued: `Waiting for <short id> — <title>`. -- The segment background uses the deleting orange in light and dark themes. Text uses normal foreground in light themes and the darkest normal background token in dark themes. - -### V2 sidebar - -- Reuse the top-right status slot and dashed-circle visual language. -- Deleting shows orange `Deleting <elapsed>`. -- Queued shows orange `Deleting (Queued)` with no timer. -- Failed shows red `Cleanup failed`. -- Use the same shared hover content as the legacy sidebar. - -### Mobile - -- Decode and retain cleanup tombstones received from the server. -- Show deleting, queued, and failed cleanup status ahead of normal thread status resolution. -- Do not add a mobile control that initiates worktree deletion. Server restart/reconnect recovery remains fully visible on mobile. - -## Validation - -- Contract encode/decode tests for every cleanup state and command/event. -- Decider tests for ownership, shared-worktree rejection, atomic initial state, retry, abandonment, and transition invariants. -- Projection/migration/query tests for persistence, startup enumeration, shell inclusion, and final removal. -- Reactor tests with controlled removal effects proving same-repository ordering, restart recovery, success, failure, queue continuation, and drain semantics. -- Projection and shell-stream tests proving tombstones survive upserts and disappear on completion/abandonment. -- Web logic tests for status priority, route fallback exclusion, and cleanup pills shared by both sidebar variants. -- Mobile presentation tests proving observe-only status priority. -- Focused lint and package typechecks, committed-range `git diff --check`, and the guarded LastCode quick-CI push gate. -- Integrated disposable-state web QA in both sidebar versions, light and dark themes, covering deleting, queued, failure, Retry, Copy details, Keep worktree, navigation fallback, and silent success. Capture before/after images; record motion if elapsed/transition behavior needs review. - -## Surface matrix - -- Web: initiates cleanup and renders/controls the full lifecycle. -- Desktop: inherits web behavior; no new Electron IPC. -- Mobile: observes lifecycle but cannot initiate worktree deletion. -- Providers: not provider-shaped; existing deletion reactor stops any provider session first. -- Contracts/server/projections: required for durable remote and multi-device behavior. -- Local, relay, and tunnel connections: use the same persisted orchestration commands and shell stream; disconnects do not own job lifetime. -- Documentation: the implementation plan and LastCode section of the user sidebar guide document this fork-only behavior. diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index 3409f8d62cb6..243f86bd4b2a 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -38,8 +38,8 @@ thread it is waiting for. Cleanup for different repositories can proceed at the If cleanup fails, the row changes to **Cleanup failed**. Select anywhere on that row to see the error and choose **Retry**, **Copy details**, or **Keep worktree**. LastCode resumes unfinished -cleanup after a server restart. Mobile shows these states but recovery actions are available only -on web and desktop. +cleanup after a server restart. On mobile, long-press a failed row to choose **Retry** or +**Keep worktree**. ## Environment artwork From 2cd0853416bb0490af86779537a183ca5c0fe547 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 09:05:40 -0700 Subject: [PATCH 08/22] fix(lastcode): preserve cleanup recovery --- .../features/threads/thread-list-items.tsx | 82 +++++++++++++++++-- .../src/orchestration/decider.delete.test.ts | 20 +++++ apps/server/src/orchestration/decider.ts | 22 +++++ 3 files changed, 115 insertions(+), 9 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index c8fe853d7687..4e4e0781b100 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -23,6 +23,7 @@ import { themeColorWithAlpha } from "../../lib/mobileTheme"; import { useThemeColor } from "../../lib/useThemeColor"; import type { PendingNewTask } from "../../state/use-pending-new-tasks"; import { terminalEnvironment } from "../../state/terminal"; +import { threadEnvironment } from "../../state/threads"; import { useAtomCommand } from "../../state/use-atom-command"; import { useThreadPr, type ThreadPr } from "../../state/use-thread-pr"; import type { HomeGroupDisplayAction } from "../home/homeListItems"; @@ -418,6 +419,11 @@ const THREAD_ROW_MENU_ACTIONS: MenuAction[] = [ { id: "delete", title: "Delete", image: "trash", attributes: { destructive: true } }, ]; +const FAILED_CLEANUP_MENU_ACTIONS: MenuAction[] = [ + { id: "retry-worktree-cleanup", title: "Retry", image: "arrow.clockwise" }, + { id: "keep-worktree", title: "Keep worktree", image: "externaldrive" }, +]; + export const ThreadListRow = memo(function ThreadListRow(props: { readonly variant: ThreadListVariant; readonly thread: EnvironmentThreadShell; @@ -459,9 +465,16 @@ export const ThreadListRow = memo(function ThreadListRow(props: { const { thread, onSelectThread, onArchiveThread, onDeleteThread, onRegenerateThreadTitle } = props; - const cleanupPending = thread.worktreeCleanup != null; + const cleanupFailed = thread.worktreeCleanup?.status === "failed"; + const cleanupPending = thread.worktreeCleanup != null && !cleanupFailed; const runningAction = thread.actionResume?.outcome === "running" ? thread.actionResume : null; const closeTerminal = useAtomCommand(terminalEnvironment.close, { reportFailure: false }); + const retryWorktreeCleanup = useAtomCommand(threadEnvironment.retryWorktreeCleanup, { + reportFailure: false, + }); + const abandonWorktreeCleanup = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { + reportFailure: false, + }); const status = resolveThreadStatus(thread); const pr = useThreadPr(thread, props.projectCwd); const timestamp = relativeTime( @@ -505,6 +518,48 @@ export const ThreadListRow = memo(function ThreadListRow(props: { ); } }, [closeTerminal, runningAction, thread.environmentId, thread.id]); + const handleRetryWorktreeCleanup = useCallback(async () => { + const result = await retryWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }); + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not retry worktree cleanup", + error instanceof Error ? error.message : "The worktree cleanup could not be retried.", + ); + } + }, [retryWorktreeCleanup, thread.environmentId, thread.id]); + const handleKeepWorktree = useCallback(() => { + Alert.alert( + "Keep worktree?", + "LastCode will stop trying to remove this worktree. You can remove it manually later.", + [ + { text: "Cancel", style: "cancel" }, + { + text: "Keep worktree", + style: "destructive", + onPress: () => { + void abandonWorktreeCleanup({ + environmentId: thread.environmentId, + input: { threadId: thread.id }, + }).then((result) => { + if (result._tag === "Failure") { + const error = Cause.squash(result.cause); + Alert.alert( + "Could not keep worktree", + error instanceof Error + ? error.message + : "The worktree cleanup could not be dismissed.", + ); + } + }); + }, + }, + ], + ); + }, [abandonWorktreeCleanup, thread.environmentId, thread.id]); const menuActions = useMemo<MenuAction[]>( () => [ THREAD_ROW_MENU_ACTIONS[0]!, @@ -540,9 +595,18 @@ export const ThreadListRow = memo(function ThreadListRow(props: { if (nativeEvent.event === "archive") handleArchive(); if (nativeEvent.event === "regenerate-title") handleRegenerateTitle(); if (nativeEvent.event === "cancel-action") void handleCancelAction(); + if (nativeEvent.event === "retry-worktree-cleanup") void handleRetryWorktreeCleanup(); + if (nativeEvent.event === "keep-worktree") handleKeepWorktree(); if (nativeEvent.event === "delete") handleDelete(); }, - [handleArchive, handleCancelAction, handleDelete, handleRegenerateTitle], + [ + handleArchive, + handleCancelAction, + handleDelete, + handleKeepWorktree, + handleRegenerateTitle, + handleRetryWorktreeCleanup, + ], ); const statusPill = effectiveStatus ? ( @@ -599,9 +663,9 @@ export const ThreadListRow = memo(function ThreadListRow(props: { accessibilityHint="Swipe left for archive and delete actions" accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" - accessibilityState={{ disabled: cleanupPending }} + accessibilityState={{ disabled: cleanupPending || cleanupFailed }} className="bg-screen" - disabled={cleanupPending} + disabled={cleanupPending || cleanupFailed} onPress={() => { close(); onSelectThread(thread); @@ -654,8 +718,8 @@ export const ThreadListRow = memo(function ThreadListRow(props: { accessibilityHint="Opens the thread" accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" - accessibilityState={{ disabled: cleanupPending, selected }} - disabled={cleanupPending} + accessibilityState={{ disabled: cleanupPending || cleanupFailed, selected }} + disabled={cleanupPending || cleanupFailed} onHoverIn={() => setHovered(true)} onHoverOut={() => setHovered(false)} onPress={() => { @@ -715,7 +779,7 @@ export const ThreadListRow = memo(function ThreadListRow(props: { return ( <ThreadSwipeable backgroundColor={backgroundColor} - enabled={!cleanupPending} + enabled={!cleanupPending && !cleanupFailed} containerStyle={ compact ? undefined : { borderRadius: SIDEBAR_ROW_RADIUS, overflow: "hidden" } } @@ -739,9 +803,9 @@ export const ThreadListRow = memo(function ThreadListRow(props: { // ControlPillMenu injects onLongPress into the row and anchors the // token-styled dropdown to it; taps and swipes are untouched. <ControlPillMenu - actions={cleanupPending ? [] : menuActions} + actions={cleanupFailed ? FAILED_CLEANUP_MENU_ACTIONS : cleanupPending ? [] : menuActions} onPressAction={handleMenuAction} - shouldOpenOnLongPress={!cleanupPending} + shouldOpenOnLongPress={cleanupFailed || !cleanupPending} > {rowContent(close)} </ControlPillMenu> diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index 9be29c84092c..b615862dfbbc 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -172,6 +172,26 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }); const afterFirst = yield* projectEvent(readModel, { ...firstEvent, sequence: 4 }); + const repeated = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-thread-delete-worktree-1-repeat"), + threadId: asThreadId("thread-delete-1"), + }, + readModel: afterFirst, + }); + const repeatedEvent = ( + Array.isArray(repeated) ? repeated[0] : repeated + ) as PlannedThreadDeletedEvent; + expect(repeatedEvent.type).toBe("thread.deleted"); + if (repeatedEvent.type !== "thread.deleted") return; + expect(repeatedEvent.payload.worktreeCleanup).toEqual(firstEvent.payload.worktreeCleanup); + const afterRepeat = yield* projectEvent(afterFirst, { ...repeatedEvent, sequence: 5 }); + expect( + afterRepeat.threads.find((thread) => thread.id === asThreadId("thread-delete-1")) + ?.worktreeCleanup, + ).toEqual(firstEvent.payload.worktreeCleanup); + const second = yield* decideOrchestrationCommand({ command: { type: "thread.delete", diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 5a9c9c6a6b3d..76278be0c58c 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -477,6 +477,28 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" threadId: command.threadId, }); const occurredAt = yield* nowIso; + + // Deletion commands can be retried after the first deleted event has + // already been projected. Preserve the tombstone, especially its + // durable worktree-cleanup state, rather than allowing a retry that + // omits deleteWorktree to clear an in-flight cleanup. + if (thread.deletedAt !== null) { + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt, + commandId: command.commandId, + })), + type: "thread.deleted", + payload: { + threadId: command.threadId, + deletedAt: thread.deletedAt, + ...(thread.worktreeCleanup === null ? {} : { worktreeCleanup: thread.worktreeCleanup }), + }, + }; + } + let worktreeCleanup: NonNullable< OrchestrationReadModel["threads"][number]["worktreeCleanup"] > | null = null; From b1577c40a2da97d19b9cd853ebe31176d72c0af4 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 09:24:48 -0700 Subject: [PATCH 09/22] fix(lastcode): retain cleanup completion retries --- .../features/threads/thread-list-items.tsx | 22 ++++++++++---- .../Layers/ThreadDeletionReactor.test.ts | 30 +++++++++++++++++-- .../Layers/ThreadDeletionReactor.ts | 16 +++++----- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-items.tsx b/apps/mobile/src/features/threads/thread-list-items.tsx index 4e4e0781b100..d2083b1545f5 100644 --- a/apps/mobile/src/features/threads/thread-list-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-items.tsx @@ -660,13 +660,18 @@ export const ThreadListRow = memo(function ThreadListRow(props: { const rowContent = (close: () => void) => compact ? ( <Pressable - accessibilityHint="Swipe left for archive and delete actions" + accessibilityHint={ + cleanupFailed + ? "Thread unavailable. Long-press for worktree recovery actions" + : "Swipe left for archive and delete actions" + } accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" - accessibilityState={{ disabled: cleanupPending || cleanupFailed }} + accessibilityState={{ disabled: cleanupPending }} className="bg-screen" - disabled={cleanupPending || cleanupFailed} + disabled={cleanupPending} onPress={() => { + if (cleanupFailed) return; close(); onSelectThread(thread); }} @@ -715,14 +720,19 @@ export const ThreadListRow = memo(function ThreadListRow(props: { </Pressable> ) : ( <Pressable - accessibilityHint="Opens the thread" + accessibilityHint={ + cleanupFailed + ? "Thread unavailable. Long-press for worktree recovery actions" + : "Opens the thread" + } accessibilityLabel={threadAccessibilityLabel} accessibilityRole="button" - accessibilityState={{ disabled: cleanupPending || cleanupFailed, selected }} - disabled={cleanupPending || cleanupFailed} + accessibilityState={{ disabled: cleanupPending, selected }} + disabled={cleanupPending} onHoverIn={() => setHovered(true)} onHoverOut={() => setHovered(false)} onPress={() => { + if (cleanupFailed) return; close(); onSelectThread(thread); }} diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 7a01b988e0bb..e7fda1fa21d4 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -14,9 +14,11 @@ import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { it as effectIt } from "@effect/vitest"; import { describe, expect, it } from "vite-plus/test"; @@ -25,6 +27,7 @@ import { ProjectionThreadRepository, type ProjectionThread, } from "../../persistence/Services/ProjectionThreads.ts"; +import { PersistenceSqlError } from "../../persistence/Errors.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; @@ -100,7 +103,7 @@ function cleanupRow( } describe("durable worktree cleanup", () => { - effectIt.live("tears down the thread before removing its worktree", () => + effectIt.live("tears down the thread before removing its worktree and retries completion", () => Effect.gen(function* () { const thread = cleanupRow( "cleanup-event", @@ -132,12 +135,30 @@ describe("durable worktree cleanup", () => { const rows = new Map([[thread.threadId, thread]]); const operations: string[] = []; const removed = yield* Deferred.make<void>(); + const completionDispatchFailed = yield* Deferred.make<void>(); + let completionDispatchAttempts = 0; const dependencies = Layer.mergeAll( Layer.mock(OrchestrationEngineService)({ streamDomainEvents: Stream.make(deletedEvent), latestSequence: Effect.succeed(1), readEvents: () => Stream.empty, dispatch: (command) => { + if ( + command.type === "thread.worktree-cleanup.update" && + command.cleanup === null && + completionDispatchAttempts++ === 0 + ) { + return Deferred.succeed(completionDispatchFailed, undefined).pipe( + Effect.andThen( + Effect.fail( + new PersistenceSqlError({ + operation: "test.dispatchCleanup", + detail: "transient persistence failure", + }), + ), + ), + ); + } if (command.type === "thread.worktree-cleanup.update") { const row = rows.get(command.threadId); if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); @@ -174,8 +195,11 @@ describe("durable worktree cleanup", () => { const reactor = yield* ThreadDeletionReactor; yield* reactor.start(); yield* Deferred.await(removed); - yield* reactor.drain; - }).pipe(Effect.provide(testLayer)); + const drain = yield* Effect.forkChild(reactor.drain); + yield* Deferred.await(completionDispatchFailed); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(drain); + }).pipe(Effect.provide(Layer.merge(testLayer, TestClock.layer()))); expect(operations).toEqual([ `stop:${thread.threadId}`, diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 4004f4db2f42..7b91afa5baff 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -4,12 +4,14 @@ import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; +import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; @@ -173,14 +175,12 @@ const make = Effect.gen(function* () { Result.isFailure(removal) && !(yield* fileSystem.exists(deleting.worktreePath)); if (Result.isSuccess(removal) || alreadyRemoved) { yield* dispatchCleanup(job.threadId, null).pipe( - Effect.retry({ times: 2 }), - Effect.catchCause((cause) => { - if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause); - return Effect.logWarning("removed worktree but could not persist cleanup completion", { - threadId: job.threadId, - worktreePath: deleting.worktreePath, - cause: Cause.pretty(cause), - }); + Effect.retry({ + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ), }), ); return; From 16cdb31da7153f72bb3dff0c342e2d99842220e3 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 09:57:49 -0700 Subject: [PATCH 10/22] fix(lastcode): serialize durable cleanup safely --- .../Layers/ThreadDeletionReactor.test.ts | 156 +++++++++++++- .../Layers/ThreadDeletionReactor.ts | 197 +++++++++++++----- apps/server/src/orchestration/Normalizer.ts | 63 ++++++ .../src/orchestration/decider.delete.test.ts | 60 ++++++ apps/server/src/orchestration/decider.ts | 28 ++- packages/contracts/src/orchestration.ts | 10 + 6 files changed, 450 insertions(+), 64 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index e7fda1fa21d4..083f69c3a598 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -13,6 +13,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as DateTime from "effect/DateTime"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -23,6 +24,8 @@ import { it as effectIt } from "@effect/vitest"; import { describe, expect, it } from "vite-plus/test"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts"; +import { ProviderAdapterProcessError } from "../../provider/Errors.ts"; import { ProjectionThreadRepository, type ProjectionThread, @@ -186,9 +189,10 @@ describe("durable worktree cleanup", () => { }), NodeServices.layer, ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); const testLayer = ThreadDeletionReactorLive.pipe( - Layer.provide(dependencies), - Layer.merge(dependencies), + Layer.provide(testDependencies), + Layer.merge(testDependencies), ); yield* Effect.gen(function* () { @@ -199,7 +203,7 @@ describe("durable worktree cleanup", () => { yield* Deferred.await(completionDispatchFailed); yield* TestClock.adjust("1 second"); yield* Fiber.join(drain); - }).pipe(Effect.provide(Layer.merge(testLayer, TestClock.layer()))); + }).pipe(Effect.provide(testLayer)); expect(operations).toEqual([ `stop:${thread.threadId}`, @@ -209,9 +213,130 @@ describe("durable worktree cleanup", () => { }), ); + effectIt.live("blocks worktree removal when teardown fails and retries failed persistence", () => + Effect.gen(function* () { + const thread = cleanupRow( + "cleanup-teardown-failed", + { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/worktrees/teardown-failed", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const deletedEvent: Extract<OrchestrationEvent, { type: "thread.deleted" }> = { + sequence: 1, + eventId: EventId.make("event-thread-deleted-teardown-failed"), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make("command-thread-deleted-teardown-failed"), + causationEventId: null, + correlationId: CommandId.make("command-thread-deleted-teardown-failed"), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; + const rows = new Map([[thread.threadId, thread]]); + const operations: string[] = []; + const teardownFailed = yield* Deferred.make<void>(); + const failureDispatchFailed = yield* Deferred.make<void>(); + let failureDispatchAttempts = 0; + const updates: Array< + Extract<OrchestrationCommand, { type: "thread.worktree-cleanup.update" }> + > = []; + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.make(deletedEvent), + latestSequence: Effect.succeed(1), + readEvents: () => Stream.empty, + dispatch: (command) => { + if ( + command.type === "thread.worktree-cleanup.update" && + command.cleanup?.status === "failed" && + failureDispatchAttempts++ === 0 + ) { + return Deferred.succeed(failureDispatchFailed, undefined).pipe( + Effect.andThen( + Effect.fail( + new PersistenceSqlError({ + operation: "test.dispatchCleanupFailure", + detail: "transient persistence failure", + }), + ), + ), + ); + } + if (command.type === "thread.worktree-cleanup.update") { + updates.push(command); + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: updates.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: () => Effect.sync(() => operations.push("remove-worktree")), + }), + Layer.mock(ProviderService)({ + stopSession: ({ threadId }) => + Effect.sync(() => operations.push(`stop:${threadId}`)).pipe( + Effect.andThen(Deferred.succeed(teardownFailed, undefined)), + Effect.andThen( + Effect.fail( + new ProviderAdapterProcessError({ + provider: "codex", + threadId: String(threadId), + detail: "provider process did not stop", + }), + ), + ), + ), + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: ({ threadId }) => Effect.sync(() => operations.push(`close:${threadId}`)), + }), + NodeServices.layer, + ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(testDependencies), + Layer.merge(testDependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + const drain = yield* Effect.forkChild(reactor.drain); + yield* Deferred.await(teardownFailed); + yield* Deferred.await(failureDispatchFailed); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(drain); + }).pipe(Effect.provide(testLayer)); + + expect(operations).toEqual([`stop:${thread.threadId}`]); + expect(failureDispatchAttempts).toBe(2); + expect(updates).toHaveLength(1); + expect(updates[0]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("ProviderAdapterProcessError"), + }); + }), + ); + effectIt.live("resumes same-repository cleanup in order and persists failures", () => Effect.gen(function* () { - const root = "/repo"; + const root = "/repo-a"; const existingWorktreePath = process.cwd(); const first = cleanupRow( "cleanup-first", @@ -227,7 +352,7 @@ describe("durable worktree cleanup", () => { "cleanup-second", { status: "queued", - repositoryRoot: `${root}/`, + repositoryRoot: "/repo-b", worktreePath: existingWorktreePath, queuedAt: "2026-08-23T00:00:01.000Z", blockedByThreadId: first.threadId, @@ -238,7 +363,7 @@ describe("durable worktree cleanup", () => { "cleanup-third", { status: "queued", - repositoryRoot: root, + repositoryRoot: "/repo-c", worktreePath: "/worktrees/third", queuedAt: "2026-08-23T00:00:02.000Z", blockedByThreadId: second.threadId, @@ -249,7 +374,7 @@ describe("durable worktree cleanup", () => { "cleanup-already-removed", { status: "deleting", - repositoryRoot: root, + repositoryRoot: "/repo-d", worktreePath: "/worktrees/already-removed", startedAt: "2026-08-23T00:00:03.000Z", }, @@ -290,6 +415,23 @@ describe("durable worktree cleanup", () => { listPendingWorktreeCleanup: () => Effect.succeed([first, second, third, fourth]), listActiveWorktreeOwners: () => Effect.succeed([activeOwner]), }), + Layer.mock(VcsDriverRegistry.VcsDriverRegistry)({ + resolve: () => + Effect.succeed({ + kind: "git" as const, + repository: { + kind: "git" as const, + rootPath: "/checkout", + metadataPath: "/shared-repository/.git", + freshness: { + source: "live-local" as const, + observedAt: DateTime.makeUnsafe("2026-08-23T00:00:00.000Z"), + expiresAt: Option.none(), + }, + }, + driver: null as never, + }), + }), Layer.mock(GitWorkflowService)({ removeWorktree: ({ path }) => Effect.gen(function* () { diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 7b91afa5baff..fd31e382ee1d 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -9,6 +9,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Schedule from "effect/Schedule"; @@ -24,6 +25,7 @@ import { type ThreadDeletionReactorShape, } from "../Services/ThreadDeletionReactor.ts"; import { forkParked } from "../../serverActivation.ts"; +import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts"; type ThreadDeletedEvent = Extract<OrchestrationEvent, { type: "thread.deleted" }>; type PendingCleanup = Exclude<ThreadWorktreeCleanup, { readonly status: "failed" }>; @@ -61,16 +63,51 @@ const make = Effect.gen(function* () { const providerService = yield* ProviderService; const terminalManager = yield* TerminalManager.TerminalManager; const fileSystem = yield* FileSystem.FileSystem; + const vcsDriverRegistry = yield* Effect.serviceOption(VcsDriverRegistry.VcsDriverRegistry); + const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const cleanupWorkersRef = yield* Ref.make<ReadonlyMap<string, DrainableWorker<CleanupJob>>>( new Map(), ); const enqueuedCleanupThreadIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); + const failedThreadTeardownIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); const serverCommandId = (tag: string) => crypto.randomUUIDv4.pipe(Effect.map((uuid) => CommandId.make(`server:${tag}:${uuid}`))); + const dispatchCleanup = Effect.fn("dispatchThreadWorktreeCleanup")(function* ( + threadId: CleanupJob["threadId"], + cleanup: ThreadWorktreeCleanup | null, + ) { + yield* orchestrationEngine.dispatch({ + type: "thread.worktree-cleanup.update", + commandId: yield* serverCommandId("worktree-cleanup-update"), + threadId, + cleanup, + }); + }); + + const cleanupPersistenceRetrySchedule = Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + ); + const dispatchCleanupWithRetry = ( + threadId: CleanupJob["threadId"], + cleanup: ThreadWorktreeCleanup | null, + ) => + dispatchCleanup(threadId, cleanup).pipe( + Effect.retry({ schedule: cleanupPersistenceRetrySchedule }), + ); + + const clearFailedThreadTeardown = (threadId: CleanupJob["threadId"]) => + Ref.update(failedThreadTeardownIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.delete(threadId); + return next; + }); + const stopProviderSession = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => logCleanupCauseUnlessInterrupted({ effect: providerService.stopSession({ threadId }), @@ -85,6 +122,14 @@ const make = Effect.gen(function* () { threadId, }); + const stopProviderSessionStrict = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + providerService + .stopSession({ threadId }) + .pipe(Effect.catchTag("ProviderSessionNotFoundError", () => Effect.void)); + + const closeThreadTerminalsStrict = (threadId: ThreadDeletedEvent["payload"]["threadId"]) => + terminalManager.close({ threadId, deleteHistory: true }); + const processThreadDeleted = Effect.fn("processThreadDeleted")(function* ( event: ThreadDeletedEvent, ) { @@ -93,38 +138,61 @@ const make = Effect.gen(function* () { yield* closeThreadTerminals(threadId); }); - const processThreadDeletedSafely = (event: ThreadDeletedEvent) => - processThreadDeleted(event).pipe( + const processThreadDeletedSafely = (event: ThreadDeletedEvent) => { + const cleanup = event.payload.worktreeCleanup; + const hasPendingWorktreeCleanup = cleanup != null && cleanup.status !== "failed"; + const teardown = hasPendingWorktreeCleanup + ? Effect.gen(function* () { + yield* stopProviderSessionStrict(event.payload.threadId); + yield* closeThreadTerminalsStrict(event.payload.threadId); + }) + : processThreadDeleted(event); + + return teardown.pipe( + Effect.tap(() => + hasPendingWorktreeCleanup ? clearFailedThreadTeardown(event.payload.threadId) : Effect.void, + ), Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { - return Effect.failCause(cause); + return Effect.interrupt; } - return Effect.logWarning("thread deletion reactor failed to process event", { - eventType: event.type, - threadId: event.payload.threadId, - cause: Cause.pretty(cause), + if (!hasPendingWorktreeCleanup || cleanup == null) { + return Effect.logWarning("thread deletion reactor failed to process event", { + eventType: event.type, + threadId: event.payload.threadId, + cause: Cause.pretty(cause), + }); + } + return Effect.gen(function* () { + yield* Ref.update(failedThreadTeardownIdsRef, (threadIds) => { + const next = new Set(threadIds); + next.add(event.payload.threadId); + return next; + }); + const failedAt = yield* nowIso; + yield* dispatchCleanupWithRetry(event.payload.threadId, { + status: "failed", + repositoryRoot: cleanup.repositoryRoot, + ...(cleanup.repositoryKey === undefined + ? {} + : { repositoryKey: cleanup.repositoryKey }), + worktreePath: cleanup.worktreePath, + startedAt: cleanup.status === "deleting" ? cleanup.startedAt : failedAt, + failedAt, + error: Cause.pretty(cause), + }); }); }), ); + }; const worker = yield* makeDrainableWorker(processThreadDeletedSafely); - const dispatchCleanup = Effect.fn("dispatchThreadWorktreeCleanup")(function* ( - threadId: CleanupJob["threadId"], - cleanup: ThreadWorktreeCleanup | null, - ) { - yield* orchestrationEngine.dispatch({ - type: "thread.worktree-cleanup.update", - commandId: yield* serverCommandId("worktree-cleanup-update"), - threadId, - cleanup, - }); - }); - const processCleanup = Effect.fn("processThreadWorktreeCleanup")(function* (job: CleanupJob) { if (job.needsTeardown) { - yield* stopProviderSession(job.threadId); - yield* closeThreadTerminals(job.threadId); + yield* stopProviderSessionStrict(job.threadId); + yield* closeThreadTerminalsStrict(job.threadId); + yield* clearFailedThreadTeardown(job.threadId); } const projected = yield* projectionThreads.getById({ threadId: job.threadId }); @@ -136,6 +204,7 @@ const make = Effect.gen(function* () { const deleting = { status: "deleting" as const, repositoryRoot: current.repositoryRoot, + ...(current.repositoryKey === undefined ? {} : { repositoryKey: current.repositoryKey }), worktreePath: current.worktreePath, startedAt: current.status === "deleting" ? current.startedAt : startedAt, }; @@ -174,15 +243,7 @@ const make = Effect.gen(function* () { const alreadyRemoved = Result.isFailure(removal) && !(yield* fileSystem.exists(deleting.worktreePath)); if (Result.isSuccess(removal) || alreadyRemoved) { - yield* dispatchCleanup(job.threadId, null).pipe( - Effect.retry({ - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), - ), - ), - }), - ); + yield* dispatchCleanupWithRetry(job.threadId, null); return; } @@ -202,23 +263,17 @@ const make = Effect.gen(function* () { const detail = Cause.pretty(cause); return Effect.gen(function* () { const failedAt = yield* nowIso; - yield* dispatchCleanup(job.threadId, { + yield* dispatchCleanupWithRetry(job.threadId, { status: "failed", repositoryRoot: job.cleanup.repositoryRoot, + ...(job.cleanup.repositoryKey === undefined + ? {} + : { repositoryKey: job.cleanup.repositoryKey }), worktreePath: job.cleanup.worktreePath, startedAt: job.cleanup.status === "deleting" ? job.cleanup.startedAt : failedAt, failedAt, error: detail, - }).pipe( - Effect.catchCause((dispatchCause) => - Effect.logWarning("thread worktree cleanup failure could not be persisted", { - threadId: job.threadId, - worktreePath: job.cleanup.worktreePath, - cleanupCause: detail, - dispatchCause: Cause.pretty(dispatchCause), - }), - ), - ); + }); }); }), ); @@ -230,10 +285,35 @@ const make = Effect.gen(function* () { return next; }); + const resolveCleanupRepositoryKey = Effect.fn("resolveCleanupRepositoryKey")(function* ( + cleanup: PendingCleanup, + ) { + const persistedKey = cleanup.repositoryKey; + + if (Option.isNone(vcsDriverRegistry)) { + return normalizeProjectPathForComparison(persistedKey ?? cleanup.repositoryRoot); + } + + const handle = yield* vcsDriverRegistry.value + .resolve({ cwd: cleanup.repositoryRoot }) + .pipe(Effect.option); + const metadataPath = Option.isNone(handle) ? null : handle.value.repository.metadataPath; + if (metadataPath === null) { + return normalizeProjectPathForComparison(persistedKey ?? cleanup.repositoryRoot); + } + const resolvedMetadataPath = path.isAbsolute(metadataPath) + ? path.normalize(metadataPath) + : path.resolve(cleanup.repositoryRoot, metadataPath); + const canonicalMetadataPath = yield* fileSystem + .realPath(resolvedMetadataPath) + .pipe(Effect.orElseSucceed(() => resolvedMetadataPath)); + return normalizeProjectPathForComparison(canonicalMetadataPath); + }); + const getCleanupWorker = Effect.fn("getThreadWorktreeCleanupWorker")(function* ( - repositoryRoot: string, + cleanup: PendingCleanup, ) { - const repositoryKey = normalizeProjectPathForComparison(repositoryRoot); + const repositoryKey = yield* resolveCleanupRepositoryKey(cleanup); const existing = (yield* Ref.get(cleanupWorkersRef)).get(repositoryKey); if (existing) return existing; const created = yield* makeDrainableWorker((job: CleanupJob) => @@ -257,7 +337,7 @@ const make = Effect.gen(function* () { }); if (!accepted) return; - const cleanupWorker = yield* getCleanupWorker(job.cleanup.repositoryRoot); + const cleanupWorker = yield* getCleanupWorker(job.cleanup); yield* cleanupWorker.enqueue(job); }); @@ -274,13 +354,15 @@ const make = Effect.gen(function* () { } if (event.type === "thread.worktree-cleanup-updated") { const cleanup = event.payload.cleanup; - return cleanup == null || cleanup.status === "failed" - ? Effect.void - : enqueueCleanup({ - threadId: event.payload.threadId, - cleanup, - needsTeardown: false, - }); + if (cleanup == null || cleanup.status === "failed") return Effect.void; + return enqueueCleanup({ + threadId: event.payload.threadId, + cleanup, + // Cleanup updates include retries after a persisted teardown failure. + // Repeating idempotent teardown is safer than relying on process-local + // memory, especially when the retry arrives after a server restart. + needsTeardown: true, + }); } return Effect.void; }; @@ -298,7 +380,18 @@ const make = Effect.gen(function* () { if (event.type === "thread.deleted") { return worker .enqueue(event) - .pipe(Effect.andThen(worker.drain), Effect.andThen(enqueueCleanupFromEvent(event))); + .pipe( + Effect.andThen(worker.drain), + Effect.andThen( + Ref.get(failedThreadTeardownIdsRef).pipe( + Effect.flatMap((failedThreadIds) => + failedThreadIds.has(event.payload.threadId) + ? Effect.void + : enqueueCleanupFromEvent(event), + ), + ), + ), + ); } return enqueueCleanupFromEvent(event); }), diff --git a/apps/server/src/orchestration/Normalizer.ts b/apps/server/src/orchestration/Normalizer.ts index 24c65900b296..ce879248b254 100644 --- a/apps/server/src/orchestration/Normalizer.ts +++ b/apps/server/src/orchestration/Normalizer.ts @@ -1,6 +1,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import { type ClientOrchestrationCommand, @@ -13,6 +14,8 @@ import { import { createAttachmentId, resolveAttachmentPath } from "../attachmentStore.ts"; import { ServerConfig } from "../config.ts"; import { parseBase64DataUrl } from "../imageMime.ts"; +import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts"; +import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; export const canonicalizeClientCommandTimestamps = ( @@ -50,8 +53,48 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const serverConfig = yield* ServerConfig; + const projectionSnapshotQuery = yield* Effect.serviceOption(ProjectionSnapshotQuery); + const vcsDriverRegistry = yield* Effect.serviceOption(VcsDriverRegistry.VcsDriverRegistry); const workspacePaths = yield* WorkspacePaths.WorkspacePaths; + const resolveGitCommonDir = (cwd: string) => + Effect.gen(function* () { + if (Option.isNone(vcsDriverRegistry)) return null; + const handle = yield* vcsDriverRegistry.value.resolve({ cwd }).pipe(Effect.option); + if (Option.isNone(handle) || handle.value.repository.metadataPath === null) { + return null; + } + const metadataPath = handle.value.repository.metadataPath; + const resolvedPath = path.isAbsolute(metadataPath) + ? path.normalize(metadataPath) + : path.resolve(cwd, metadataPath); + return yield* fileSystem + .realPath(resolvedPath) + .pipe(Effect.orElseSucceed(() => resolvedPath)); + }); + + const resolveProjectRepositoryKey = (projectId: string) => + Effect.gen(function* () { + if (Option.isNone(projectionSnapshotQuery)) return null; + const readModel = yield* projectionSnapshotQuery.value + .getCommandReadModel() + .pipe(Effect.option); + if (Option.isNone(readModel)) return null; + const project = readModel.value.projects.find((candidate) => candidate.id === projectId); + return project === undefined ? null : yield* resolveGitCommonDir(project.workspaceRoot); + }); + + const resolveThreadDeleteRepositoryKey = (threadId: string) => + Effect.gen(function* () { + if (Option.isNone(projectionSnapshotQuery)) return null; + const readModel = yield* projectionSnapshotQuery.value + .getCommandReadModel() + .pipe(Effect.option); + if (Option.isNone(readModel)) return null; + const thread = readModel.value.threads.find((candidate) => candidate.id === threadId); + return thread === undefined ? null : yield* resolveProjectRepositoryKey(thread.projectId); + }); + const normalizeProjectWorkspaceRoot = (workspaceRoot: string) => workspacePaths.normalizeWorkspaceRoot(workspaceRoot).pipe( Effect.mapError( @@ -100,6 +143,26 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) => } satisfies OrchestrationCommand; } + if (canonicalCommand.type === "thread.delete" && canonicalCommand.deleteWorktree === true) { + const repositoryKey = yield* resolveThreadDeleteRepositoryKey(canonicalCommand.threadId); + const { repositoryKey: _clientRepositoryKey, ...commandWithoutRepositoryKey } = + canonicalCommand; + return { + ...commandWithoutRepositoryKey, + ...(repositoryKey === null ? {} : { repositoryKey }), + } satisfies OrchestrationCommand; + } + + if (canonicalCommand.type === "project.delete" && canonicalCommand.force === true) { + const repositoryKey = yield* resolveProjectRepositoryKey(canonicalCommand.projectId); + const { repositoryKey: _clientRepositoryKey, ...commandWithoutRepositoryKey } = + canonicalCommand; + return { + ...commandWithoutRepositoryKey, + ...(repositoryKey === null ? {} : { repositoryKey }), + } satisfies OrchestrationCommand; + } + if (canonicalCommand.type !== "thread.turn.start") { return canonicalCommand as OrchestrationCommand; } diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index b615862dfbbc..6ee066dd8839 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -213,6 +213,66 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }), ); + it.effect("queues cleanups from different checkouts that share a Git common directory", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const firstReadModel = { + ...seeded, + threads: seeded.threads.map((thread, index) => ({ + ...thread, + projectId: index === 1 ? asProjectId("project-delete-sibling") : thread.projectId, + branch: `sibling-cleanup-${index + 1}`, + worktreePath: `/tmp/sibling-worktrees/cleanup-${index + 1}`, + })), + projects: [ + ...seeded.projects, + { + ...seeded.projects[0]!, + id: asProjectId("project-delete-sibling"), + workspaceRoot: "/tmp/project-delete-sibling", + }, + ], + }; + + const first = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-sibling-delete-1"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + repositoryKey: "/tmp/shared-repository/.git", + }, + readModel: firstReadModel, + }); + const firstEvent = (Array.isArray(first) ? first[0] : first) as PlannedThreadDeletedEvent; + const afterFirst = yield* projectEvent(firstReadModel, { ...firstEvent, sequence: 4 }); + + const second = yield* decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-sibling-delete-2"), + threadId: asThreadId("thread-delete-2"), + deleteWorktree: true, + repositoryKey: "/tmp/shared-repository/.git", + }, + readModel: afterFirst, + }); + const secondEvent = (Array.isArray(second) ? second[0] : second) as PlannedThreadDeletedEvent; + + expect(firstEvent.payload.worktreeCleanup).toMatchObject({ + status: "deleting", + repositoryRoot: "/tmp/project-delete", + repositoryKey: "/tmp/shared-repository/.git", + }); + expect(secondEvent.payload.worktreeCleanup).toMatchObject({ + status: "queued", + repositoryRoot: "/tmp/project-delete-sibling", + repositoryKey: "/tmp/shared-repository/.git", + blockedByThreadId: asThreadId("thread-delete-1"), + }); + }), + ); + it.effect("refuses to delete a worktree still owned by another live thread", () => Effect.gen(function* () { const seeded = yield* seedReadModel; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 76278be0c58c..b4088d09e68b 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -174,10 +174,10 @@ function worktreeCleanupTimestamp( function findWorktreeCleanupBlocker( readModel: OrchestrationReadModel, - repositoryRoot: string, + repositoryKey: string, exceptThreadId?: string, ) { - const normalizedRoot = normalizeProjectPathForComparison(repositoryRoot); + const normalizedKey = normalizeProjectPathForComparison(repositoryKey); return readModel.threads .filter((candidate) => { const cleanup = candidate.worktreeCleanup; @@ -185,7 +185,8 @@ function findWorktreeCleanupBlocker( candidate.id !== exceptThreadId && cleanup != null && cleanup.status !== "failed" && - normalizeProjectPathForComparison(cleanup.repositoryRoot) === normalizedRoot + normalizeProjectPathForComparison(cleanup.repositoryKey ?? cleanup.repositoryRoot) === + normalizedKey ); }) .toSorted((left, right) => { @@ -400,6 +401,9 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" type: "thread.delete", commandId: command.commandId, threadId: thread.id, + ...(command.repositoryKey === undefined + ? {} + : { repositoryKey: command.repositoryKey }), }), ), { @@ -528,18 +532,25 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" detail: `Worktree '${thread.worktreePath}' is still used by thread '${sharedThread.id}'.`, }); } - const blocker = findWorktreeCleanupBlocker(readModel, project.workspaceRoot, thread.id); + const repositoryKey = command.repositoryKey; + const blocker = findWorktreeCleanupBlocker( + readModel, + repositoryKey ?? project.workspaceRoot, + thread.id, + ); worktreeCleanup = blocker === undefined ? { status: "deleting", repositoryRoot: project.workspaceRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), worktreePath: thread.worktreePath, startedAt: occurredAt, } : { status: "queued", repositoryRoot: project.workspaceRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), worktreePath: thread.worktreePath, queuedAt: occurredAt, blockedByThreadId: blocker.id, @@ -571,18 +582,25 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" }); } const occurredAt = yield* nowIso; - const blocker = findWorktreeCleanupBlocker(readModel, cleanup.repositoryRoot, thread.id); + const repositoryKey = cleanup.repositoryKey; + const blocker = findWorktreeCleanupBlocker( + readModel, + repositoryKey ?? cleanup.repositoryRoot, + thread.id, + ); const nextCleanup = blocker === undefined ? { status: "deleting" as const, repositoryRoot: cleanup.repositoryRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), worktreePath: cleanup.worktreePath, startedAt: occurredAt, } : { status: "queued" as const, repositoryRoot: cleanup.repositoryRoot, + ...(repositoryKey === undefined ? {} : { repositoryKey }), worktreePath: cleanup.worktreePath, queuedAt: occurredAt, blockedByThreadId: blocker.id, diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 737b6b3aea01..d99651cf1791 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -464,6 +464,12 @@ export type ThreadAnnotation = typeof ThreadAnnotation.Type; const ThreadWorktreeCleanupBase = { repositoryRoot: TrimmedNonEmptyString, + /** + * Canonical Git common directory used to serialize worktree operations. + * Optional for cleanup rows written before repository identity was added; + * those rows fall back to repositoryRoot when selecting a blocker/worker. + */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), worktreePath: TrimmedNonEmptyString, } as const; @@ -791,6 +797,8 @@ const ProjectDeleteCommand = Schema.Struct({ commandId: CommandId, projectId: ProjectId, force: Schema.optional(Schema.Boolean), + /** Resolved by command normalization for forced worktree cleanup. */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), }); const ThreadCreateCommand = Schema.Struct({ @@ -814,6 +822,8 @@ const ThreadDeleteCommand = Schema.Struct({ commandId: CommandId, threadId: ThreadId, deleteWorktree: Schema.optional(Schema.Boolean), + /** Resolved by command normalization from the thread's project checkout. */ + repositoryKey: Schema.optional(TrimmedNonEmptyString), }); const ThreadWorktreeCleanupRetryCommand = Schema.Struct({ From bf9061c4d056f90cf678c5b89a5c2eb5d7456592 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 10:18:08 -0700 Subject: [PATCH 11/22] fix(web): preserve cleanup recovery paths --- .../WorktreeCleanupFailureDialog.test.tsx | 100 ++++++++++++++++++ .../WorktreeCleanupFailureDialog.tsx | 20 +++- apps/web/src/hooks/useThreadActions.test.ts | 31 +++++- apps/web/src/hooks/useThreadActions.ts | 82 ++++++++++++-- 4 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 apps/web/src/components/WorktreeCleanupFailureDialog.test.tsx diff --git a/apps/web/src/components/WorktreeCleanupFailureDialog.test.tsx b/apps/web/src/components/WorktreeCleanupFailureDialog.test.tsx new file mode 100644 index 000000000000..abf8e26544a4 --- /dev/null +++ b/apps/web/src/components/WorktreeCleanupFailureDialog.test.tsx @@ -0,0 +1,100 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import type { SidebarThreadSummary } from "../types"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const testState = vi.hoisted(() => ({ + copy: vi.fn(), + copyDetails: undefined as (() => void) | undefined, + copyOptions: undefined as { onError?: (error: unknown) => void } | undefined, + toast: vi.fn(), +})); + +vi.mock("../hooks/useCopyToClipboard", () => ({ + useCopyToClipboard: (options: { onError?: (error: unknown) => void }) => { + testState.copyOptions = options; + return { copyToClipboard: (value: string) => testState.copy(value) }; + }, +})); +vi.mock("../state/threads", () => ({ + threadEnvironment: { + abandonWorktreeCleanup: Symbol("abandonWorktreeCleanup"), + retryWorktreeCleanup: Symbol("retryWorktreeCleanup"), + }, +})); +vi.mock("../state/use-atom-command", () => ({ + useAtomCommand: () => vi.fn(), +})); +vi.mock("./ui/toast", () => ({ + stackedThreadToast: (toast: unknown) => toast, + toastManager: { add: testState.toast }, +})); +vi.mock("./ui/button", () => ({ + Button: (props: { children?: unknown; onClick?: () => void }) => { + if (props.children === "Copy details") testState.copyDetails = props.onClick; + return null; + }, +})); +vi.mock("./ui/dialog", () => { + const passthrough = ({ children }: { children?: unknown }) => children; + return { + Dialog: passthrough, + DialogDescription: passthrough, + DialogFooter: passthrough, + DialogHeader: passthrough, + DialogPanel: passthrough, + DialogPopup: passthrough, + DialogTitle: passthrough, + }; +}); + +import { WorktreeCleanupFailureDialog } from "./WorktreeCleanupFailureDialog"; + +const failedThread = { + environmentId: "environment-test", + id: "thread-test", + title: "Deleted thread", + worktreeCleanup: { + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/deleted", + startedAt: "2026-08-24T10:00:00.000Z", + failedAt: "2026-08-24T10:01:00.000Z", + error: "permission denied", + }, +} as SidebarThreadSummary; + +function renderDialog(): void { + renderToStaticMarkup( + <WorktreeCleanupFailureDialog thread={failedThread} open onOpenChange={() => undefined} />, + ); +} + +describe("WorktreeCleanupFailureDialog", () => { + beforeEach(() => { + vi.clearAllMocks(); + testState.copyDetails = undefined; + testState.copyOptions = undefined; + }); + + it("routes Copy details through the guarded clipboard helper", () => { + renderDialog(); + + testState.copyDetails?.(); + + expect(testState.copy).toHaveBeenCalledWith( + expect.stringContaining("Worktree: /repo-worktrees/deleted"), + ); + }); + + it("reports clipboard failures instead of throwing from the click handler", () => { + renderDialog(); + + testState.copyOptions?.onError?.(new Error("Clipboard API is unavailable")); + + expect(testState.toast).toHaveBeenCalledWith({ + type: "error", + title: "Could not copy worktree cleanup details", + description: "Clipboard API is unavailable", + }); + }); +}); diff --git a/apps/web/src/components/WorktreeCleanupFailureDialog.tsx b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx index 490cf39e3198..9b7695bcd991 100644 --- a/apps/web/src/components/WorktreeCleanupFailureDialog.tsx +++ b/apps/web/src/components/WorktreeCleanupFailureDialog.tsx @@ -2,6 +2,7 @@ import type { SidebarThreadSummary } from "../types"; import { threadEnvironment } from "../state/threads"; import { useAtomCommand } from "../state/use-atom-command"; import { ensureLocalApi } from "../localApi"; +import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; import { Button } from "./ui/button"; import { Dialog, @@ -12,6 +13,7 @@ import { DialogPopup, DialogTitle, } from "./ui/dialog"; +import { stackedThreadToast, toastManager } from "./ui/toast"; export function WorktreeCleanupFailureDialog(props: { thread: SidebarThreadSummary; @@ -22,6 +24,18 @@ export function WorktreeCleanupFailureDialog(props: { const abandon = useAtomCommand(threadEnvironment.abandonWorktreeCleanup, { reportFailure: true, }); + const { copyToClipboard } = useCopyToClipboard({ + target: "worktree cleanup details", + onError: (error) => { + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Could not copy worktree cleanup details", + description: error instanceof Error ? error.message : "An error occurred while copying.", + }), + ); + }, + }); const cleanup = props.thread.worktreeCleanup; if (cleanup?.status !== "failed") return null; @@ -76,11 +90,7 @@ export function WorktreeCleanupFailureDialog(props: { <Button type="button" variant="destructive-outline" onClick={() => void keepWorktree()}> Keep worktree </Button> - <Button - type="button" - variant="outline" - onClick={() => void navigator.clipboard.writeText(details)} - > + <Button type="button" variant="outline" onClick={() => copyToClipboard(details)}> Copy details </Button> <Button diff --git a/apps/web/src/hooks/useThreadActions.test.ts b/apps/web/src/hooks/useThreadActions.test.ts index c5385211591f..f3c4c6dbc949 100644 --- a/apps/web/src/hooks/useThreadActions.test.ts +++ b/apps/web/src/hooks/useThreadActions.test.ts @@ -1,7 +1,7 @@ import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { ThreadArchiveBlockedError } from "./useThreadActions"; +import { shouldDeleteWorktreeClientSide, ThreadArchiveBlockedError } from "./useThreadActions"; describe("ThreadArchiveBlockedError", () => { it("keeps the blocked thread context with the fixed message", () => { @@ -17,3 +17,32 @@ describe("ThreadArchiveBlockedError", () => { expect(error.message).toBe("Cannot archive a running thread."); }); }); + +describe("shouldDeleteWorktreeClientSide", () => { + it("keeps the legacy client-side cleanup path for older servers", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: true, + supportsDurableWorktreeCleanup: false, + }), + ).toBe(true); + }); + + it("leaves cleanup to the durable server path when supported", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: true, + supportsDurableWorktreeCleanup: true, + }), + ).toBe(false); + }); + + it("does not remove a worktree when the user keeps it", () => { + expect( + shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree: false, + supportsDurableWorktreeCleanup: false, + }), + ).toBe(false); + }); +}); diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index f050c2de9ac7..ccb847772778 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -4,7 +4,7 @@ import { scopeThreadRef, scopedThreadKey, } from "@t3tools/client-runtime/environment"; -import { settlePromise } from "@t3tools/client-runtime/state/runtime"; +import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { canSettle, canSnooze, threadWokeAt } from "@t3tools/client-runtime/state/thread-settled"; import { EnvironmentId, type ScopedThreadRef, ThreadId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -17,6 +17,7 @@ import { getFallbackThreadIdAfterDelete, pinOrderKeyBetween } from "../component import { useComposerDraftStore } from "../composerDraftStore"; import { terminalEnvironment } from "../state/terminal"; import { threadEnvironment } from "../state/threads"; +import { vcsEnvironment } from "../state/vcs"; import { useNewThreadHandler } from "./useHandleNewThread"; import { refreshArchivedThreadsForEnvironment } from "../lib/archivedThreadsState"; import { readLocalApi } from "../localApi"; @@ -35,6 +36,7 @@ import { useTerminalUiStateStore } from "../terminalUiStateStore"; import { useUiStateStore } from "../uiStateStore"; import { buildThreadRouteParams, resolveThreadRouteRef } from "../threadRoutes"; import { formatWorktreePathForDisplay, getOrphanedWorktreePathForThread } from "../worktreeCleanup"; +import { stackedThreadToast, toastManager } from "../components/ui/toast"; import { useClientSettings } from "./useSettings"; import { useAtomCommand } from "../state/use-atom-command"; @@ -50,6 +52,13 @@ export class ThreadArchiveBlockedError extends Schema.TaggedErrorClass<ThreadArc } } +export function shouldDeleteWorktreeClientSide(input: { + readonly shouldDeleteWorktree: boolean; + readonly supportsDurableWorktreeCleanup: boolean; +}): boolean { + return input.shouldDeleteWorktree && !input.supportsDurableWorktreeCleanup; +} + export class ThreadSettlementUnsupportedError extends Schema.TaggedErrorClass<ThreadSettlementUnsupportedError>()( "ThreadSettlementUnsupportedError", { @@ -167,6 +176,12 @@ export function useThreadActions() { reportFailure: false, }); const stopThreadSession = useAtomCommand(threadEnvironment.stopSession); + const removeWorktree = useAtomCommand(vcsEnvironment.removeWorktree, { + reportFailure: false, + }); + const refreshVcsStatus = useAtomCommand(vcsEnvironment.refreshStatus, { + reportFailure: false, + }); const sidebarThreadSortOrder = useClientSettings((settings) => settings.sidebarThreadSortOrder); const confirmThreadDelete = useClientSettings((settings) => settings.confirmThreadDelete); const clearComposerDraftForThread = useComposerDraftStore((store) => store.clearDraftThread); @@ -305,10 +320,10 @@ export function useThreadActions() { const displayWorktreePath = orphanedWorktreePath ? formatWorktreePathForDisplay(orphanedWorktreePath) : null; - const canDeleteWorktree = - orphanedWorktreePath !== null && - threadProject !== null && - readEnvironmentSupportsWorktreeCleanup(threadRef.environmentId); + const supportsDurableWorktreeCleanup = readEnvironmentSupportsWorktreeCleanup( + threadRef.environmentId, + ); + const canDeleteWorktree = orphanedWorktreePath !== null && threadProject !== null; const localApi = readLocalApi(); let shouldDeleteWorktree = false; if (canDeleteWorktree && localApi) { @@ -356,7 +371,9 @@ export function useThreadActions() { environmentId: threadRef.environmentId, input: { threadId: threadRef.threadId, - ...(shouldDeleteWorktree ? { deleteWorktree: true } : {}), + ...(shouldDeleteWorktree && supportsDurableWorktreeCleanup + ? { deleteWorktree: true } + : {}), }, }); if (deleteResult._tag === "Failure") { @@ -406,6 +423,57 @@ export function useThreadActions() { } } + if ( + !shouldDeleteWorktreeClientSide({ + shouldDeleteWorktree, + supportsDurableWorktreeCleanup, + }) || + !orphanedWorktreePath || + !threadProject + ) { + return deleteResult; + } + + const removeResult = await removeWorktree({ + environmentId: threadRef.environmentId, + input: { + cwd: threadProject.workspaceRoot, + path: orphanedWorktreePath, + force: true, + }, + }); + const refreshResult = + removeResult._tag === "Success" + ? await refreshVcsStatus({ + environmentId: threadRef.environmentId, + input: { cwd: threadProject.workspaceRoot }, + }) + : null; + const cleanupFailure = + removeResult._tag === "Failure" + ? removeResult + : refreshResult?._tag === "Failure" + ? refreshResult + : null; + if (cleanupFailure) { + const error = squashAtomCommandFailure(cleanupFailure); + const message = error instanceof Error ? error.message : "Unknown error removing worktree."; + console.error("Failed to remove orphaned worktree after thread deletion", { + threadId: threadRef.threadId, + projectCwd: threadProject.workspaceRoot, + worktreePath: orphanedWorktreePath, + error, + }); + toastManager.add( + stackedThreadToast({ + type: "error", + title: "Thread deleted, but worktree removal failed", + description: `Could not remove ${displayWorktreePath ?? orphanedWorktreePath}. ${message}`, + }), + ); + return cleanupFailure; + } + return deleteResult; }, [ @@ -415,6 +483,8 @@ export function useThreadActions() { closeTerminal, deleteThreadMutation, getCurrentRouteThreadRef, + refreshVcsStatus, + removeWorktree, router, resolveThreadTarget, sidebarThreadSortOrder, From 795bd93b84a57a8559becd1564fd210d1361aec2 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 10:34:39 -0700 Subject: [PATCH 12/22] fix(mobile): surface cleanup tombstones in v2 --- .../features/threads/thread-list-v2-items.tsx | 6 +- .../src/features/threads/threadListV2.test.ts | 87 ++++++++++++++++++- .../src/features/threads/threadListV2.ts | 7 ++ .../features/threads/threadPresentation.ts | 17 ++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/apps/mobile/src/features/threads/thread-list-v2-items.tsx b/apps/mobile/src/features/threads/thread-list-v2-items.tsx index 126ff341db30..55557c2a72ea 100644 --- a/apps/mobile/src/features/threads/thread-list-v2-items.tsx +++ b/apps/mobile/src/features/threads/thread-list-v2-items.tsx @@ -30,6 +30,7 @@ import { useThreadPr } from "../../state/use-thread-pr"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; import { useAppearancePreferences } from "../settings/appearance/AppearancePreferencesProvider"; import { buildThreadTitleRegenerationMenuItems } from "./thread-title-regeneration-menu"; +import { resolveWorktreeCleanupStatus } from "./threadPresentation"; import { resolveThreadListV2SnoozeMenuSelection, resolveThreadListV2SnoozeGateExpiryMs, @@ -443,7 +444,10 @@ export const ThreadListV2Row = memo(function ThreadListV2Row(props: { const selected = props.selected === true; const status = resolveThreadListV2Status(thread); - const statusLabel = STATUS_LABEL_BY_STATUS[status]; + const cleanupStatus = resolveWorktreeCleanupStatus(thread); + const statusLabel = cleanupStatus + ? { label: cleanupStatus.label, className: cleanupStatus.textClassName } + : STATUS_LABEL_BY_STATUS[status]; const timeLabel = threadTimeLabel(thread); const handleDelete = useCallback(() => onDeleteThread(thread), [onDeleteThread, thread]); diff --git a/apps/mobile/src/features/threads/threadListV2.test.ts b/apps/mobile/src/features/threads/threadListV2.test.ts index 0edfdc65afb6..c272c9414bf1 100644 --- a/apps/mobile/src/features/threads/threadListV2.test.ts +++ b/apps/mobile/src/features/threads/threadListV2.test.ts @@ -24,7 +24,7 @@ import { resolveThreadListV2SwipeActions, sortThreadsForListV2, } from "./threadListV2"; -import { resolveThreadStatus } from "./threadPresentation"; +import { resolveThreadStatus, resolveWorktreeCleanupStatus } from "./threadPresentation"; const environmentId = EnvironmentId.make("environment-1"); @@ -144,6 +144,10 @@ describe("resolveThreadListV2Status", () => { label: "Deleting", pulse: false, }); + expect(resolveWorktreeCleanupStatus(deleting)).toMatchObject({ + kind: "cleanup-deleting", + label: "Deleting", + }); expect( resolveThreadStatus({ ...deleting, @@ -157,6 +161,18 @@ describe("resolveThreadListV2Status", () => { }, }), ).toMatchObject({ kind: "cleanup-failed", label: "Cleanup failed" }); + expect( + resolveWorktreeCleanupStatus({ + ...deleting, + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/cleanup", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("blocking"), + }, + }), + ).toMatchObject({ kind: "cleanup-queued", label: "Deleting (Queued)" }); }); it("prioritizes approval over a running session", () => { @@ -347,6 +363,75 @@ describe("sortThreadsForListV2", () => { }); describe("buildThreadListV2Items", () => { + it("keeps all cleanup tombstones in the visible active block", () => { + const cleanupTombstones = [ + makeThread({ + id: ThreadId.make("cleanup-queued"), + title: "Queued cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "queued", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/queued", + queuedAt: NOW, + blockedByThreadId: ThreadId.make("cleanup-blocker"), + }, + }), + makeThread({ + id: ThreadId.make("cleanup-deleting"), + title: "Deleting cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/deleting", + startedAt: NOW, + }, + }), + makeThread({ + id: ThreadId.make("cleanup-failed"), + title: "Failed cleanup", + pinnedAt: NOW, + settledOverride: "settled", + settledAt: NOW, + snoozedUntil: "2026-06-03T09:00:00.000Z", + snoozedAt: NOW, + worktreeCleanup: { + status: "failed", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/failed", + startedAt: NOW, + failedAt: NOW, + error: "permission denied", + }, + }), + ]; + const layout = buildThreadListV2Items({ + threads: cleanupTombstones, + environmentId: null, + searchQuery: "", + now: NOW, + }); + + expect(layout.items.map((item) => item.thread.id)).toEqual([ + "cleanup-deleting", + "cleanup-failed", + "cleanup-queued", + ]); + expect(layout.items.map((item) => item.variant)).toEqual(["card", "card", "card"]); + expect(layout.items.map((item) => item.pinned)).toEqual([false, false, false]); + expect(layout.snoozedCount).toBe(0); + expect(layout.settledCount).toBe(0); + }); + it("keeps a merged thread active when auto-settle on merge is off", () => { const merged = makeThread({ id: ThreadId.make("merged"), title: "Merged" }); const layout = buildThreadListV2Items({ diff --git a/apps/mobile/src/features/threads/threadListV2.ts b/apps/mobile/src/features/threads/threadListV2.ts index eb2e77997a22..261d97b9a06d 100644 --- a/apps/mobile/src/features/threads/threadListV2.ts +++ b/apps/mobile/src/features/threads/threadListV2.ts @@ -401,6 +401,13 @@ export function buildThreadListV2Items(input: { const supportsSnooze = input.snoozeEnvironmentIds?.has(thread.environmentId) ?? true; const changeRequest = input.changeRequestByKey?.get(`${thread.environmentId}:${thread.id}`) ?? null; + // Cleanup tombstones are deleted-thread recovery state, not lifecycle + // state. Keep them in the immediately visible active block regardless of + // stale snooze, settle, or pin metadata retained on the thread. + if (thread.worktreeCleanup != null) { + active.push(thread); + continue; + } // Snooze outranks settlement and pinning until the thread wakes. if (supportsSnooze && effectiveSnoozed(thread, { now: snoozeNow })) { snoozed.push(thread); diff --git a/apps/mobile/src/features/threads/threadPresentation.ts b/apps/mobile/src/features/threads/threadPresentation.ts index 52bab7f93cb2..0bd0cfa51371 100644 --- a/apps/mobile/src/features/threads/threadPresentation.ts +++ b/apps/mobile/src/features/threads/threadPresentation.ts @@ -179,3 +179,20 @@ export function resolveThreadStatus( return null; } + +/** + * Returns the durable cleanup status when a thread is being deleted. Mobile + * list variants use this shared presentation so cleanup state cannot fall + * through to the ordinary agent-status labels. + */ +export function resolveWorktreeCleanupStatus( + thread: EnvironmentThreadShell, +): ThreadStatusPresentation | null { + if (thread.worktreeCleanup == null) return null; + const status = resolveThreadStatus(thread); + return status?.kind === "cleanup-failed" || + status?.kind === "cleanup-queued" || + status?.kind === "cleanup-deleting" + ? status + : null; +} From eece7167e94de99280b7d968b9e466fa588a4c20 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 10:59:00 -0700 Subject: [PATCH 13/22] fix(server): retire idle cleanup workers --- .../Layers/ThreadDeletionReactor.test.ts | 167 ++++++++++++++++++ .../Layers/ThreadDeletionReactor.ts | 67 +++++-- packages/shared/src/DrainableWorker.ts | 15 +- 3 files changed, 236 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 083f69c3a598..a002a2826a62 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -18,6 +18,7 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { it as effectIt } from "@effect/vitest"; @@ -105,6 +106,54 @@ function cleanupRow( }; } +function deletedEventFor( + thread: ProjectionThread, + eventId: string, + sequence: number, +): Extract<OrchestrationEvent, { type: "thread.deleted" }> { + return { + sequence, + eventId: EventId.make(eventId), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.deleted", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make(`${eventId}-command`), + causationEventId: null, + correlationId: CommandId.make(`${eventId}-correlation`), + metadata: {}, + payload: { + threadId: thread.threadId, + deletedAt: "2026-08-23T00:00:00.000Z", + worktreeCleanup: thread.worktreeCleanup ?? undefined, + }, + }; +} + +function cleanupUpdatedEventFor( + thread: ProjectionThread, + eventId: string, + sequence: number, +): Extract<OrchestrationEvent, { type: "thread.worktree-cleanup-updated" }> { + return { + sequence, + eventId: EventId.make(eventId), + aggregateKind: "thread", + aggregateId: thread.threadId, + type: "thread.worktree-cleanup-updated", + occurredAt: "2026-08-23T00:00:00.000Z", + commandId: CommandId.make(`${eventId}-command`), + causationEventId: null, + correlationId: CommandId.make(`${eventId}-correlation`), + metadata: {}, + payload: { + threadId: thread.threadId, + cleanup: thread.worktreeCleanup!, + updatedAt: "2026-08-23T00:00:00.000Z", + }, + }; +} + describe("durable worktree cleanup", () => { effectIt.live("tears down the thread before removing its worktree and retries completion", () => Effect.gen(function* () { @@ -334,6 +383,124 @@ describe("durable worktree cleanup", () => { }), ); + effectIt.live("retires idle workers and serializes jobs on a recreated worker", () => + Effect.gen(function* () { + const root = "/repo"; + const first = cleanupRow( + "cleanup-retire-first", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-first", + startedAt: "2026-08-23T00:00:00.000Z", + }, + "2026-08-23T00:00:00.000Z", + ); + const second = cleanupRow( + "cleanup-retire-second", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-second", + startedAt: "2026-08-23T00:00:01.000Z", + }, + "2026-08-23T00:00:01.000Z", + ); + const third = cleanupRow( + "cleanup-retire-third", + { + status: "deleting", + repositoryRoot: root, + worktreePath: "/worktrees/retire-third", + startedAt: "2026-08-23T00:00:02.000Z", + }, + "2026-08-23T00:00:02.000Z", + ); + const rows = new Map([ + [first.threadId, first], + [second.threadId, second], + [third.threadId, third], + ]); + const events = yield* PubSub.unbounded<OrchestrationEvent>(); + const firstRemoved = yield* Deferred.make<void>(); + const secondStarted = yield* Deferred.make<void>(); + const thirdStarted = yield* Deferred.make<void>(); + const releaseSecond = yield* Deferred.make<void>(); + const removalOrder: string[] = []; + let activeRemovals = 0; + let maxActiveRemovals = 0; + const dependencies = Layer.mergeAll( + Layer.mock(OrchestrationEngineService)({ + streamDomainEvents: Stream.fromPubSub(events), + latestSequence: Effect.succeed(0), + readEvents: () => Stream.empty, + dispatch: (command) => { + if (command.type === "thread.worktree-cleanup.update") { + const row = rows.get(command.threadId); + if (row) rows.set(command.threadId, { ...row, worktreeCleanup: command.cleanup }); + } + return Effect.succeed({ sequence: removalOrder.length }); + }, + }), + Layer.mock(ProjectionThreadRepository)({ + getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), + listPendingWorktreeCleanup: () => Effect.succeed([]), + listActiveWorktreeOwners: () => Effect.succeed([]), + }), + Layer.mock(GitWorkflowService)({ + removeWorktree: ({ path }) => + Effect.gen(function* () { + activeRemovals += 1; + maxActiveRemovals = Math.max(maxActiveRemovals, activeRemovals); + if (path === first.worktreePath) { + yield* Deferred.succeed(firstRemoved, undefined); + } else if (path === second.worktreePath) { + yield* Deferred.succeed(secondStarted, undefined); + yield* Deferred.await(releaseSecond); + } else if (path === third.worktreePath) { + yield* Deferred.succeed(thirdStarted, undefined); + } + removalOrder.push(path); + }).pipe(Effect.ensuring(Effect.sync(() => (activeRemovals -= 1)))), + }), + Layer.mock(ProviderService)({ + stopSession: () => Effect.void, + }), + Layer.mock(TerminalManager.TerminalManager)({ + close: () => Effect.void, + }), + NodeServices.layer, + ); + const testDependencies = Layer.merge(TestClock.layer(), dependencies); + const testLayer = ThreadDeletionReactorLive.pipe( + Layer.provide(testDependencies), + Layer.merge(testDependencies), + ); + + yield* Effect.gen(function* () { + const reactor = yield* ThreadDeletionReactor; + yield* reactor.start(); + yield* Effect.yieldNow; + yield* PubSub.publish(events, deletedEventFor(first, "event-retire-first", 1)); + yield* Deferred.await(firstRemoved); + yield* reactor.drain; + + // The first repository worker has been idle long enough to retire. + yield* TestClock.adjust("1 minute"); + yield* PubSub.publish(events, cleanupUpdatedEventFor(second, "event-retire-second", 2)); + yield* PubSub.publish(events, cleanupUpdatedEventFor(third, "event-retire-third", 3)); + yield* Deferred.await(secondStarted); + expect(yield* Deferred.isDone(thirdStarted)).toBe(false); + yield* Deferred.succeed(releaseSecond, undefined); + yield* Deferred.await(thirdStarted); + yield* reactor.drain; + }).pipe(Effect.provide(testLayer)); + + expect(removalOrder).toEqual([first.worktreePath, second.worktreePath, third.worktreePath]); + expect(maxActiveRemovals).toBe(1); + }), + ); + effectIt.live("resumes same-repository cleanup in order and persists failures", () => Effect.gen(function* () { const root = "/repo-a"; diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index fd31e382ee1d..4c202703feef 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -13,6 +13,7 @@ import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; @@ -34,6 +35,13 @@ type CleanupJob = { readonly cleanup: PendingCleanup; readonly needsTeardown: boolean; }; +type CleanupWorkerEntry = { + readonly repositoryKey: string; + readonly worker: DrainableWorker<CleanupJob>; + readonly generation: Ref.Ref<number>; +}; + +const CLEANUP_WORKER_IDLE_TIMEOUT = Duration.minutes(1); export const logCleanupCauseUnlessInterrupted = <R, E>({ effect, @@ -66,9 +74,8 @@ const make = Effect.gen(function* () { const vcsDriverRegistry = yield* Effect.serviceOption(VcsDriverRegistry.VcsDriverRegistry); const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; - const cleanupWorkersRef = yield* Ref.make<ReadonlyMap<string, DrainableWorker<CleanupJob>>>( - new Map(), - ); + const cleanupWorkersRef = yield* Ref.make<ReadonlyMap<string, CleanupWorkerEntry>>(new Map()); + const cleanupWorkersMutex = yield* Semaphore.make(1); const enqueuedCleanupThreadIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); const failedThreadTeardownIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); @@ -319,13 +326,46 @@ const make = Effect.gen(function* () { const created = yield* makeDrainableWorker((job: CleanupJob) => processCleanupSafely(job).pipe(Effect.ensuring(removeEnqueuedCleanupThreadId(job.threadId))), ); - return yield* Ref.modify(cleanupWorkersRef, (workers) => { - const current = workers.get(repositoryKey); - if (current) return [current, workers] as const; + const entry: CleanupWorkerEntry = { + repositoryKey, + worker: created, + generation: yield* Ref.make(0), + }; + yield* Ref.update(cleanupWorkersRef, (workers) => { const next = new Map(workers); - next.set(repositoryKey, created); - return [created, next] as const; + next.set(repositoryKey, entry); + return next; }); + yield* Effect.forkScoped( + Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CLEANUP_WORKER_IDLE_TIMEOUT); + const generation = yield* cleanupWorkersMutex.withPermit(Ref.get(entry.generation)); + // Drain outside the global mutex so a long cleanup for one + // repository cannot block unrelated repositories from enqueueing. + yield* entry.worker.drain; + const retired = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const current = (yield* Ref.get(cleanupWorkersRef)).get(repositoryKey); + if (current !== entry || (yield* Ref.get(entry.generation)) !== generation) { + return false; + } + yield* Ref.update(cleanupWorkersRef, (workers) => { + const next = new Map(workers); + if (next.get(repositoryKey) === entry) next.delete(repositoryKey); + return next; + }); + return true; + }), + ); + if (retired) { + yield* entry.worker.shutdown; + return; + } + } + }), + ); + return entry; }); const enqueueCleanup = Effect.fn("enqueueThreadWorktreeCleanup")(function* (job: CleanupJob) { @@ -337,8 +377,13 @@ const make = Effect.gen(function* () { }); if (!accepted) return; - const cleanupWorker = yield* getCleanupWorker(job.cleanup); - yield* cleanupWorker.enqueue(job); + yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const entry = yield* getCleanupWorker(job.cleanup); + yield* Ref.update(entry.generation, (generation) => generation + 1); + yield* entry.worker.enqueue(job); + }), + ); }); const enqueueCleanupFromEvent = (event: OrchestrationEvent) => { @@ -369,7 +414,7 @@ const make = Effect.gen(function* () { const cleanupDrain = Effect.gen(function* () { const workers = yield* Ref.get(cleanupWorkersRef); - yield* Effect.forEach(workers.values(), (cleanupWorker) => cleanupWorker.drain, { + yield* Effect.forEach(workers.values(), (entry) => entry.worker.drain, { concurrency: "unbounded", }); }); diff --git a/packages/shared/src/DrainableWorker.ts b/packages/shared/src/DrainableWorker.ts index de40ec5e36b8..4eb61c7bccf4 100644 --- a/packages/shared/src/DrainableWorker.ts +++ b/packages/shared/src/DrainableWorker.ts @@ -10,6 +10,7 @@ */ import * as Scope from "effect/Scope"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as TxQueue from "effect/TxQueue"; import * as TxRef from "effect/TxRef"; @@ -26,6 +27,14 @@ export interface DrainableWorker<A> { * Resolves when the queue is empty and the worker is idle (not processing). */ readonly drain: Effect.Effect<void>; + + /** + * Stop the worker after its current queue has been drained. + * + * Callers that coordinate access to a worker may use this to retire idle + * keyed workers before their parent scope closes. + */ + readonly shutdown: Effect.Effect<void>; } /** @@ -44,7 +53,7 @@ export const makeDrainableWorker = <A, E, R>( const queue = yield* Effect.acquireRelease(TxQueue.unbounded<A>(), TxQueue.shutdown); const outstanding = yield* TxRef.make(0); - yield* TxQueue.take(queue).pipe( + const workerFiber = yield* TxQueue.take(queue).pipe( Effect.tap((a) => Effect.ensuring( process(a), @@ -66,5 +75,7 @@ export const makeDrainableWorker = <A, E, R>( Effect.tx, ); - return { enqueue, drain } satisfies DrainableWorker<A>; + const shutdown = Fiber.interrupt(workerFiber).pipe(Effect.asVoid); + + return { enqueue, drain, shutdown } satisfies DrainableWorker<A>; }); From 3795fc448fad21dd8e7e33fbe46903e05a43cdfe Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 11:19:06 -0700 Subject: [PATCH 14/22] fix(shared): release retired worker resources --- packages/shared/src/DrainableWorker.test.ts | 23 +++++++++++++++++++++ packages/shared/src/DrainableWorker.ts | 22 +++++++++++++------- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/shared/src/DrainableWorker.test.ts b/packages/shared/src/DrainableWorker.test.ts index 8e4c654e2e4c..f87e859dbf45 100644 --- a/packages/shared/src/DrainableWorker.test.ts +++ b/packages/shared/src/DrainableWorker.test.ts @@ -54,4 +54,27 @@ describe("makeDrainableWorker", () => { }), ), ); + + it.live("shuts down the worker and queue without retaining later work", () => + Effect.scoped( + Effect.gen(function* () { + const processed: string[] = []; + const worker = yield* makeDrainableWorker((item: string) => + Effect.sync(() => processed.push(item)), + ); + + yield* worker.enqueue("before-shutdown"); + yield* worker.drain; + yield* worker.shutdown; + + // Offers after retirement are rejected by the closed queue and must + // not make drain wait forever. Closing the parent scope repeats the + // queue finalizer, proving shutdown is safe to call twice. + yield* worker.enqueue("after-shutdown"); + yield* worker.drain; + + expect(processed).toEqual(["before-shutdown"]); + }), + ), + ); }); diff --git a/packages/shared/src/DrainableWorker.ts b/packages/shared/src/DrainableWorker.ts index 4eb61c7bccf4..d126c1e9f335 100644 --- a/packages/shared/src/DrainableWorker.ts +++ b/packages/shared/src/DrainableWorker.ts @@ -10,7 +10,7 @@ */ import * as Scope from "effect/Scope"; import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; +import * as Exit from "effect/Exit"; import * as TxQueue from "effect/TxQueue"; import * as TxRef from "effect/TxRef"; @@ -50,10 +50,13 @@ export const makeDrainableWorker = <A, E, R>( process: (item: A) => Effect.Effect<void, E, R>, ): Effect.Effect<DrainableWorker<A>, never, Scope.Scope | R> => Effect.gen(function* () { - const queue = yield* Effect.acquireRelease(TxQueue.unbounded<A>(), TxQueue.shutdown); + const workerScope = yield* Scope.make("sequential"); + yield* Effect.addFinalizer(() => Scope.close(workerScope, Exit.void).pipe(Effect.ignore)); + const queue = yield* TxQueue.unbounded<A>(); + yield* Scope.addFinalizer(workerScope, TxQueue.shutdown(queue).pipe(Effect.asVoid)); const outstanding = yield* TxRef.make(0); - const workerFiber = yield* TxQueue.take(queue).pipe( + yield* TxQueue.take(queue).pipe( Effect.tap((a) => Effect.ensuring( process(a), @@ -61,7 +64,7 @@ export const makeDrainableWorker = <A, E, R>( ), ), Effect.forever, - Effect.forkScoped, + Effect.forkIn(workerScope), ); const drain: DrainableWorker<A>["drain"] = TxRef.get(outstanding).pipe( @@ -69,13 +72,18 @@ export const makeDrainableWorker = <A, E, R>( Effect.tx, ); - const enqueue = (element: A): Effect.Effect<boolean, never, never> => + const enqueue = (element: A): Effect.Effect<void, never, never> => TxQueue.offer(queue, element).pipe( - Effect.tap(() => TxRef.update(outstanding, (n) => n + 1)), + Effect.tap((accepted) => + accepted ? TxRef.update(outstanding, (n) => n + 1) : Effect.void, + ), + Effect.asVoid, Effect.tx, ); - const shutdown = Fiber.interrupt(workerFiber).pipe(Effect.asVoid); + // Closing the child scope interrupts the worker and shuts down the queue. + // The parent scope also closes it, and Scope.close is idempotent. + const shutdown = Scope.close(workerScope, Exit.void).pipe(Effect.ignore); return { enqueue, drain, shutdown } satisfies DrainableWorker<A>; }); From 15cb7544f97c7eebe8aebca9001c837a6ca51df5 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 11:40:12 -0700 Subject: [PATCH 15/22] fix(web): clean up archived thread worktrees --- .../components/settings/SettingsPanels.tsx | 23 ++++--- apps/web/src/hooks/useThreadActions.test.ts | 57 +++++++++++++++- apps/web/src/hooks/useThreadActions.ts | 68 +++++++++++++++++-- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 8397e8904b90..7d22de14b502 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -7,10 +7,10 @@ import { type BackgroundActivityProfile, type DesktopUpdateChannel, ProviderDriverKind, - type ScopedThreadRef, type SidebarProjectGroupingMode, } from "@t3tools/contracts"; import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; import { isAtomCommandInterrupted, settlePromise, @@ -2516,7 +2516,8 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null ); const handleArchivedThreadContextMenu = useCallback( - async (threadRef: ScopedThreadRef, position: { x: number; y: number }) => { + async (thread: EnvironmentThreadShell, position: { x: number; y: number }) => { + const threadRef = scopeThreadRef(thread.environmentId, thread.id); const api = readLocalApi(); if (!api) return; const clicked = await api.contextMenu.show( @@ -2545,7 +2546,10 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null } if (clicked === "delete") { - const result = await confirmAndDeleteThread(threadRef); + const archivedThreads = archivedGroups + .filter((group) => group.project.environmentId === thread.environmentId) + .flatMap((group) => group.threads); + const result = await confirmAndDeleteThread(threadRef, { archivedThreads }); if (result._tag === "Success") { refreshArchivedThreads(); } else if (!isAtomCommandInterrupted(result)) { @@ -2560,7 +2564,7 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null } } }, - [confirmAndDeleteThread, refreshArchivedThreads, unarchiveThread], + [archivedGroups, confirmAndDeleteThread, refreshArchivedThreads, unarchiveThread], ); return ( @@ -2618,13 +2622,10 @@ export function ArchivedThreadsPanel({ projectKey }: { projectKey: string | null event.preventDefault(); void (async () => { const result = await settlePromise(() => - handleArchivedThreadContextMenu( - scopeThreadRef(thread.environmentId, thread.id), - { - x: event.clientX, - y: event.clientY, - }, - ), + handleArchivedThreadContextMenu(thread, { + x: event.clientX, + y: event.clientY, + }), ); if (result._tag === "Failure") { const error = squashAtomCommandFailure(result); diff --git a/apps/web/src/hooks/useThreadActions.test.ts b/apps/web/src/hooks/useThreadActions.test.ts index f3c4c6dbc949..24a65cb08fa5 100644 --- a/apps/web/src/hooks/useThreadActions.test.ts +++ b/apps/web/src/hooks/useThreadActions.test.ts @@ -1,7 +1,13 @@ import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { shouldDeleteWorktreeClientSide, ThreadArchiveBlockedError } from "./useThreadActions"; +import { getOrphanedWorktreePathForThread } from "../worktreeCleanup"; +import { + collectThreadDeleteCandidates, + resolveThreadTargetWithArchivedFallback, + shouldDeleteWorktreeClientSide, + ThreadArchiveBlockedError, +} from "./useThreadActions"; describe("ThreadArchiveBlockedError", () => { it("keeps the blocked thread context with the fixed message", () => { @@ -46,3 +52,52 @@ describe("shouldDeleteWorktreeClientSide", () => { ).toBe(false); }); }); + +describe("resolveThreadTargetWithArchivedFallback", () => { + const target = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }; + const archivedThread = { + environmentId: target.environmentId, + id: target.threadId, + worktreePath: "/tmp/archived-worktree", + }; + + it("lets archived settings deletion use the archived shell", () => { + expect(resolveThreadTargetWithArchivedFallback(target, null, [archivedThread])).toEqual({ + thread: archivedThread, + threadRef: target, + }); + }); + + it("rejects a fallback shell from another target", () => { + expect( + resolveThreadTargetWithArchivedFallback(target, null, [ + { ...archivedThread, id: ThreadId.make("other-thread") }, + ]), + ).toBeNull(); + }); +}); + +describe("collectThreadDeleteCandidates", () => { + it("keeps an archived sibling in orphan detection", () => { + const environmentId = EnvironmentId.make("environment-1"); + const target = { + environmentId, + id: ThreadId.make("thread-1"), + worktreePath: "/tmp/shared-worktree", + }; + const sibling = { + environmentId, + id: ThreadId.make("thread-2"), + worktreePath: "/tmp/shared-worktree", + }; + + const candidates = collectThreadDeleteCandidates([], target, [sibling]); + + expect(candidates).toHaveLength(2); + expect(candidates.map((thread) => thread.id)).toEqual(["thread-2", "thread-1"]); + expect(getOrphanedWorktreePathForThread(candidates, target.id)).toBeNull(); + }); +}); diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index ccb847772778..eebe36cb95ce 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -6,6 +6,7 @@ import { } from "@t3tools/client-runtime/environment"; import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { canSettle, canSnooze, threadWokeAt } from "@t3tools/client-runtime/state/thread-settled"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; import { EnvironmentId, type ScopedThreadRef, ThreadId } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Schema from "effect/Schema"; @@ -59,6 +60,48 @@ export function shouldDeleteWorktreeClientSide(input: { return input.shouldDeleteWorktree && !input.supportsDurableWorktreeCleanup; } +export type DeleteThreadOptions = { + readonly deletedThreadKeys?: ReadonlySet<string>; + /** Shells supplied by archived-thread views, which are outside the active store. */ + readonly archivedThreads?: ReadonlyArray<EnvironmentThreadShell>; +}; + +export function resolveThreadTargetWithArchivedFallback< + T extends Pick<EnvironmentThreadShell, "environmentId" | "id">, +>( + target: ScopedThreadRef, + activeThread: T | null, + archivedThreads: ReadonlyArray<T> | undefined, +): { readonly thread: T; readonly threadRef: ScopedThreadRef } | null { + const candidate = + activeThread ?? + archivedThreads?.find( + (thread) => thread.environmentId === target.environmentId && thread.id === target.threadId, + ); + if ( + candidate === undefined || + candidate.environmentId !== target.environmentId || + candidate.id !== target.threadId + ) { + return null; + } + return { thread: candidate, threadRef: target }; +} + +export function collectThreadDeleteCandidates< + T extends Pick<EnvironmentThreadShell, "environmentId" | "id" | "worktreePath">, +>( + activeThreads: ReadonlyArray<T>, + targetThread: T, + archivedThreads: ReadonlyArray<T>, +): ReadonlyArray<T> { + const candidates = new Map<string, T>(); + for (const thread of [...activeThreads, ...archivedThreads, targetThread]) { + candidates.set(`${thread.environmentId}:${thread.id}`, thread); + } + return [...candidates.values()]; +} + export class ThreadSettlementUnsupportedError extends Schema.TaggedErrorClass<ThreadSettlementUnsupportedError>()( "ThreadSettlementUnsupportedError", { @@ -278,8 +321,12 @@ export function useThreadActions() { ); const deleteThread = useCallback( - async (target: ScopedThreadRef, opts: { deletedThreadKeys?: ReadonlySet<string> } = {}) => { - const resolved = resolveThreadTarget(target); + async (target: ScopedThreadRef, opts: DeleteThreadOptions = {}) => { + const resolved = resolveThreadTargetWithArchivedFallback( + target, + resolveThreadTarget(target)?.thread ?? null, + opts.archivedThreads, + ); if (!resolved) { // Thread not in main store (e.g. archived thread) — dispatch delete directly. const result = await deleteThreadMutation({ @@ -292,10 +339,15 @@ export function useThreadActions() { return result; } const { thread, threadRef } = resolved; - const threads = readEnvironmentThreadRefs(threadRef.environmentId).flatMap((ref) => { + const activeThreads = readEnvironmentThreadRefs(threadRef.environmentId).flatMap((ref) => { const shell = readThreadShell(ref); return shell === null ? [] : [shell]; }); + const threads = collectThreadDeleteCandidates( + activeThreads, + thread, + opts.archivedThreads ?? [], + ); const threadProject = readProject({ environmentId: threadRef.environmentId, projectId: thread.projectId, @@ -691,9 +743,13 @@ export function useThreadActions() { ); const confirmAndDeleteThread = useCallback( - async (target: ScopedThreadRef) => { + async (target: ScopedThreadRef, opts: Pick<DeleteThreadOptions, "archivedThreads"> = {}) => { const localApi = readLocalApi(); - const resolved = resolveThreadTarget(target); + const resolved = resolveThreadTargetWithArchivedFallback( + target, + resolveThreadTarget(target)?.thread ?? null, + opts.archivedThreads, + ); if (confirmThreadDelete && localApi) { const title = resolved?.thread.title ?? "this thread"; @@ -714,7 +770,7 @@ export function useThreadActions() { } } - return deleteThread(target); + return deleteThread(target, opts); }, [confirmThreadDelete, deleteThread, resolveThreadTarget], ); From 0e9934bddc5a73f4a337be7d065f668ea3891620 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:09:55 -0700 Subject: [PATCH 16/22] fix(server): preserve active project workspaces --- .../Layers/ThreadDeletionReactor.test.ts | 46 ++++++++++++++++++- .../Layers/ThreadDeletionReactor.ts | 17 +++++++ .../src/orchestration/decider.delete.test.ts | 38 +++++++++++++++ apps/server/src/orchestration/decider.ts | 11 +++++ 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index a002a2826a62..2ce6352c833d 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -27,6 +27,7 @@ import { describe, expect, it } from "vite-plus/test"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; import * as VcsDriverRegistry from "../../vcs/VcsDriverRegistry.ts"; import { ProviderAdapterProcessError } from "../../provider/Errors.ts"; +import { ProjectionProjectRepository } from "../../persistence/Services/ProjectionProjects.ts"; import { ProjectionThreadRepository, type ProjectionThread, @@ -223,6 +224,9 @@ describe("durable worktree cleanup", () => { listPendingWorktreeCleanup: () => Effect.succeed([]), listActiveWorktreeOwners: () => Effect.succeed([]), }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), Layer.mock(GitWorkflowService)({ removeWorktree: ({ path }) => Effect.sync(() => operations.push(`remove:${path}`)).pipe( @@ -334,6 +338,9 @@ describe("durable worktree cleanup", () => { listPendingWorktreeCleanup: () => Effect.succeed([]), listActiveWorktreeOwners: () => Effect.succeed([]), }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), Layer.mock(GitWorkflowService)({ removeWorktree: () => Effect.sync(() => operations.push("remove-worktree")), }), @@ -447,6 +454,9 @@ describe("durable worktree cleanup", () => { listPendingWorktreeCleanup: () => Effect.succeed([]), listActiveWorktreeOwners: () => Effect.succeed([]), }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => Effect.succeed([]), + }), Layer.mock(GitWorkflowService)({ removeWorktree: ({ path }) => Effect.gen(function* () { @@ -547,6 +557,16 @@ describe("durable worktree cleanup", () => { }, "2026-08-23T00:00:03.000Z", ); + const fifth = cleanupRow( + "cleanup-active-project-root", + { + status: "deleting", + repositoryRoot: "/repo-e", + worktreePath: "/worktrees/active-project-root", + startedAt: "2026-08-23T00:00:04.000Z", + }, + "2026-08-23T00:00:04.000Z", + ); const activeOwner = { threadId: ThreadId.make("active-owner"), worktreePath: third.worktreePath ?? "/worktrees/third", @@ -556,6 +576,7 @@ describe("durable worktree cleanup", () => { [second.threadId, second], [third.threadId, third], [fourth.threadId, fourth], + [fifth.threadId, fifth], ]); const removals: string[] = []; const operations: string[] = []; @@ -579,9 +600,25 @@ describe("durable worktree cleanup", () => { }), Layer.mock(ProjectionThreadRepository)({ getById: ({ threadId }) => Effect.succeed(Option.fromUndefinedOr(rows.get(threadId))), - listPendingWorktreeCleanup: () => Effect.succeed([first, second, third, fourth]), + listPendingWorktreeCleanup: () => Effect.succeed([first, second, third, fourth, fifth]), listActiveWorktreeOwners: () => Effect.succeed([activeOwner]), }), + Layer.mock(ProjectionProjectRepository)({ + listAll: () => + Effect.succeed([ + { + projectId: ProjectId.make("active-project"), + title: "Active project", + workspaceRoot: fifth.worktreePath!, + defaultModelSelection: null, + defaultThreadEnvMode: null, + scripts: [], + createdAt: "2026-08-23T00:00:00.000Z", + updatedAt: "2026-08-23T00:00:00.000Z", + deletedAt: null, + }, + ]), + }), Layer.mock(VcsDriverRegistry.VcsDriverRegistry)({ resolve: () => Effect.succeed({ @@ -659,6 +696,8 @@ describe("durable worktree cleanup", () => { `stop:${fourth.threadId}`, `close:${fourth.threadId}`, "remove:/worktrees/already-removed", + `stop:${fifth.threadId}`, + `close:${fifth.threadId}`, ]); expect( updates.map((command) => [command.threadId, command.cleanup?.status ?? "complete"]), @@ -669,6 +708,7 @@ describe("durable worktree cleanup", () => { [third.threadId, "deleting"], [third.threadId, "failed"], [fourth.threadId, "complete"], + [fifth.threadId, "failed"], ]); expect(updates[2]?.cleanup).toMatchObject({ status: "failed", @@ -679,6 +719,10 @@ describe("durable worktree cleanup", () => { error: expect.stringContaining("active-owner"), }); expect(updates[5]?.cleanup).toBeNull(); + expect(updates[6]?.cleanup).toMatchObject({ + status: "failed", + error: expect.stringContaining("active-project"), + }); }), ); }); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 4c202703feef..84ef2375fa84 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -17,6 +17,7 @@ import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { GitWorkflowService } from "../../git/GitWorkflowService.ts"; +import { ProjectionProjectRepository } from "../../persistence/Services/ProjectionProjects.ts"; import { ProjectionThreadRepository } from "../../persistence/Services/ProjectionThreads.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import * as TerminalManager from "../../terminal/Manager.ts"; @@ -67,6 +68,7 @@ export const logCleanupCauseUnlessInterrupted = <R, E>({ const make = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngineService; const gitWorkflow = yield* GitWorkflowService; + const projectionProjects = yield* ProjectionProjectRepository; const projectionThreads = yield* ProjectionThreadRepository; const providerService = yield* ProviderService; const terminalManager = yield* TerminalManager.TerminalManager; @@ -220,6 +222,21 @@ const make = Effect.gen(function* () { } const normalizedWorktreePath = normalizeProjectPathForComparison(deleting.worktreePath); + const activeProject = (yield* projectionProjects.listAll()).find( + (candidate) => + candidate.deletedAt === null && + normalizeProjectPathForComparison(candidate.workspaceRoot) === normalizedWorktreePath, + ); + if (activeProject !== undefined) { + const failedAt = yield* nowIso; + yield* dispatchCleanup(job.threadId, { + ...deleting, + status: "failed", + failedAt, + error: `Worktree '${deleting.worktreePath}' is now used as the workspace root of active project '${activeProject.projectId}'.`, + }); + return; + } const activeOwner = (yield* projectionThreads.listActiveWorktreeOwners()).find( (candidate) => candidate.threadId !== job.threadId && diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index 6ee066dd8839..eaec21c218f3 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -273,6 +273,44 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { }), ); + it.effect("rejects deleting a worktree registered as an active project root", () => + Effect.gen(function* () { + const seeded = yield* seedReadModel; + const worktreePath = "/tmp/project-delete-worktrees/active-project"; + const readModel = { + ...seeded, + threads: seeded.threads.map((thread) => + thread.id === asThreadId("thread-delete-1") + ? { ...thread, branch: "active-project", worktreePath } + : thread, + ), + projects: [ + ...seeded.projects, + { + ...seeded.projects[0]!, + id: asProjectId("project-active-worktree"), + workspaceRoot: worktreePath, + }, + ], + }; + + const error = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "thread.delete", + commandId: asCommandId("cmd-delete-active-project-worktree"), + threadId: asThreadId("thread-delete-1"), + deleteWorktree: true, + }, + readModel, + }), + ); + + expect(error.message).toContain("project-active-worktree"); + expect(error.message).toContain("workspace root"); + }), + ); + it.effect("refuses to delete a worktree still owned by another live thread", () => Effect.gen(function* () { const seeded = yield* seedReadModel; diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index b4088d09e68b..2e220e091a44 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -519,6 +519,17 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" projectId: thread.projectId, }); const normalizedWorktreePath = normalizeProjectPathForComparison(thread.worktreePath); + const sharedProject = readModel.projects.find( + (candidate) => + candidate.deletedAt === null && + normalizeProjectPathForComparison(candidate.workspaceRoot) === normalizedWorktreePath, + ); + if (sharedProject !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Worktree '${thread.worktreePath}' is still used as the workspace root of project '${sharedProject.id}'.`, + }); + } const sharedThread = readModel.threads.find( (candidate) => candidate.id !== thread.id && From 0956a4c02a5b27e06184872c288c64520eb7cf0a Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:24:54 -0700 Subject: [PATCH 17/22] fix(web): include archived worktree owners --- apps/web/src/hooks/useThreadActions.test.ts | 30 +++++++++++++++++++ apps/web/src/hooks/useThreadActions.ts | 32 +++++++++++++++++---- apps/web/src/lib/archivedThreadsState.ts | 30 ++++++++++++++++++- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/apps/web/src/hooks/useThreadActions.test.ts b/apps/web/src/hooks/useThreadActions.test.ts index 24a65cb08fa5..764fc622d726 100644 --- a/apps/web/src/hooks/useThreadActions.test.ts +++ b/apps/web/src/hooks/useThreadActions.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vite-plus/test"; import { getOrphanedWorktreePathForThread } from "../worktreeCleanup"; import { collectThreadDeleteCandidates, + resolveArchivedThreadsForDelete, resolveThreadTargetWithArchivedFallback, shouldDeleteWorktreeClientSide, ThreadArchiveBlockedError, @@ -101,3 +102,32 @@ describe("collectThreadDeleteCandidates", () => { expect(getOrphanedWorktreePathForThread(candidates, target.id)).toBeNull(); }); }); + +describe("resolveArchivedThreadsForDelete", () => { + it("loads archived owners for a normal worktree deletion", async () => { + const archivedThread = { + environmentId: EnvironmentId.make("environment-1"), + id: ThreadId.make("archived-owner"), + worktreePath: "/tmp/shared-worktree", + }; + + await expect( + resolveArchivedThreadsForDelete({ + worktreePath: "/tmp/shared-worktree", + load: async () => [archivedThread], + }), + ).resolves.toEqual([archivedThread]); + }); + + it("uses supplied archived shells without loading them again", async () => { + const load = () => Promise.reject(new Error("should not load")); + + await expect( + resolveArchivedThreadsForDelete({ + archivedThreads: [], + worktreePath: "/tmp/shared-worktree", + load, + }), + ).resolves.toEqual([]); + }); +}); diff --git a/apps/web/src/hooks/useThreadActions.ts b/apps/web/src/hooks/useThreadActions.ts index eebe36cb95ce..e89adea9ad54 100644 --- a/apps/web/src/hooks/useThreadActions.ts +++ b/apps/web/src/hooks/useThreadActions.ts @@ -20,7 +20,10 @@ import { terminalEnvironment } from "../state/terminal"; import { threadEnvironment } from "../state/threads"; import { vcsEnvironment } from "../state/vcs"; import { useNewThreadHandler } from "./useHandleNewThread"; -import { refreshArchivedThreadsForEnvironment } from "../lib/archivedThreadsState"; +import { + loadArchivedThreadsForEnvironment, + refreshArchivedThreadsForEnvironment, +} from "../lib/archivedThreadsState"; import { readLocalApi } from "../localApi"; import { readEnvironmentSupportsPinning, @@ -102,6 +105,16 @@ export function collectThreadDeleteCandidates< return [...candidates.values()]; } +export function resolveArchivedThreadsForDelete<T>(input: { + readonly archivedThreads?: ReadonlyArray<T>; + readonly worktreePath: string | null; + readonly load: () => Promise<ReadonlyArray<T>>; +}): Promise<ReadonlyArray<T>> { + if (input.archivedThreads !== undefined) return Promise.resolve(input.archivedThreads); + if (input.worktreePath === null) return Promise.resolve([]); + return input.load(); +} + export class ThreadSettlementUnsupportedError extends Schema.TaggedErrorClass<ThreadSettlementUnsupportedError>()( "ThreadSettlementUnsupportedError", { @@ -339,15 +352,22 @@ export function useThreadActions() { return result; } const { thread, threadRef } = resolved; + const archivedThreadsResult = await settlePromise(() => + resolveArchivedThreadsForDelete({ + ...(opts.archivedThreads === undefined ? {} : { archivedThreads: opts.archivedThreads }), + worktreePath: thread.worktreePath, + load: () => loadArchivedThreadsForEnvironment(threadRef.environmentId), + }), + ); + if (archivedThreadsResult._tag === "Failure") { + return archivedThreadsResult; + } + const archivedThreads = archivedThreadsResult.value; const activeThreads = readEnvironmentThreadRefs(threadRef.environmentId).flatMap((ref) => { const shell = readThreadShell(ref); return shell === null ? [] : [shell]; }); - const threads = collectThreadDeleteCandidates( - activeThreads, - thread, - opts.archivedThreads ?? [], - ); + const threads = collectThreadDeleteCandidates(activeThreads, thread, archivedThreads); const threadProject = readProject({ environmentId: threadRef.environmentId, projectId: thread.projectId, diff --git a/apps/web/src/lib/archivedThreadsState.ts b/apps/web/src/lib/archivedThreadsState.ts index 4087b24ac330..0f6945f44855 100644 --- a/apps/web/src/lib/archivedThreadsState.ts +++ b/apps/web/src/lib/archivedThreadsState.ts @@ -4,7 +4,10 @@ import { createArchivedThreadSnapshotsAtomFamily, makeArchivedThreadsEnvironmentKey, } from "@t3tools/client-runtime/state/threads"; -import type { EnvironmentId } from "@t3tools/contracts"; +import type { EnvironmentId, OrchestrationShellSnapshot } from "@t3tools/contracts"; +import type { EnvironmentThreadShell } from "@t3tools/client-runtime/state/shell"; +import * as Cause from "effect/Cause"; +import { AsyncResult } from "effect/unstable/reactivity"; import { useCallback, useMemo } from "react"; import { @@ -34,6 +37,31 @@ export function refreshArchivedThreadsForEnvironment(environmentId: EnvironmentI appAtomRegistry.refresh(archivedSnapshotAtom(environmentId)); } +/** Refresh and read the archived shells before destructive ownership checks. */ +export function loadArchivedThreadsForEnvironment( + environmentId: EnvironmentId, +): Promise<ReadonlyArray<EnvironmentThreadShell>> { + const atom = archivedSnapshotAtom(environmentId); + appAtomRegistry.refresh(atom); + + return new Promise((resolve, reject) => { + let unsubscribe = () => {}; + const settle = (result: AsyncResult.AsyncResult<OrchestrationShellSnapshot, unknown>) => { + if (result.waiting) return; + if (result._tag === "Success") { + unsubscribe(); + resolve(result.value.threads.map((thread) => ({ ...thread, environmentId }))); + } else if (result._tag === "Failure") { + unsubscribe(); + reject(Cause.squash(result.cause)); + } + }; + + unsubscribe = appAtomRegistry.subscribe(atom, settle); + settle(appAtomRegistry.get(atom)); + }); +} + export function useArchivedThreadSnapshots(environmentIds: ReadonlyArray<EnvironmentId>): { readonly snapshots: ReadonlyArray<ArchivedSnapshotEntry>; readonly error: string | null; From 75884be38b64123eeed4228c9c63c6c007937d01 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:46:52 -0700 Subject: [PATCH 18/22] fix(server): resolve cleanup ownership paths --- .../Layers/ThreadDeletionReactor.test.ts | 18 +++++++-- .../Layers/ThreadDeletionReactor.ts | 37 ++++++++++++++----- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 2ce6352c833d..6eb90d2123e9 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -13,11 +13,13 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as DateTime from "effect/DateTime"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; @@ -513,6 +515,16 @@ describe("durable worktree cleanup", () => { effectIt.live("resumes same-repository cleanup in order and persists failures", () => Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const activeProjectRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-active-project-root-", + }); + const aliasParent = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-active-project-alias-", + }); + const activeProjectAlias = path.join(aliasParent, "workspace"); + yield* fileSystem.symlink(activeProjectRoot, activeProjectAlias); const root = "/repo-a"; const existingWorktreePath = process.cwd(); const first = cleanupRow( @@ -562,7 +574,7 @@ describe("durable worktree cleanup", () => { { status: "deleting", repositoryRoot: "/repo-e", - worktreePath: "/worktrees/active-project-root", + worktreePath: activeProjectAlias, startedAt: "2026-08-23T00:00:04.000Z", }, "2026-08-23T00:00:04.000Z", @@ -609,7 +621,7 @@ describe("durable worktree cleanup", () => { { projectId: ProjectId.make("active-project"), title: "Active project", - workspaceRoot: fifth.worktreePath!, + workspaceRoot: activeProjectRoot, defaultModelSelection: null, defaultThreadEnvMode: null, scripts: [], @@ -723,6 +735,6 @@ describe("durable worktree cleanup", () => { status: "failed", error: expect.stringContaining("active-project"), }); - }), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 84ef2375fa84..3ea6809431b7 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -82,6 +82,11 @@ const make = Effect.gen(function* () { const failedThreadTeardownIdsRef = yield* Ref.make<ReadonlySet<string>>(new Set()); const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); + const canonicalPathForComparison = (value: string) => + fileSystem.realPath(value).pipe( + Effect.map(normalizeProjectPathForComparison), + Effect.orElseSucceed(() => normalizeProjectPathForComparison(value)), + ); const serverCommandId = (tag: string) => crypto.randomUUIDv4.pipe(Effect.map((uuid) => CommandId.make(`server:${tag}:${uuid}`))); @@ -221,12 +226,19 @@ const make = Effect.gen(function* () { yield* dispatchCleanup(job.threadId, deleting); } - const normalizedWorktreePath = normalizeProjectPathForComparison(deleting.worktreePath); - const activeProject = (yield* projectionProjects.listAll()).find( - (candidate) => - candidate.deletedAt === null && - normalizeProjectPathForComparison(candidate.workspaceRoot) === normalizedWorktreePath, + const normalizedWorktreePath = yield* canonicalPathForComparison(deleting.worktreePath); + const activeProjects = yield* Effect.forEach( + yield* projectionProjects.listAll(), + (project) => + canonicalPathForComparison(project.workspaceRoot).pipe( + Effect.map((workspaceRoot) => ({ project, workspaceRoot })), + ), + { concurrency: "unbounded" }, ); + const activeProject = activeProjects.find( + ({ project, workspaceRoot }) => + project.deletedAt === null && workspaceRoot === normalizedWorktreePath, + )?.project; if (activeProject !== undefined) { const failedAt = yield* nowIso; yield* dispatchCleanup(job.threadId, { @@ -237,11 +249,18 @@ const make = Effect.gen(function* () { }); return; } - const activeOwner = (yield* projectionThreads.listActiveWorktreeOwners()).find( - (candidate) => - candidate.threadId !== job.threadId && - normalizeProjectPathForComparison(candidate.worktreePath) === normalizedWorktreePath, + const activeOwners = yield* Effect.forEach( + yield* projectionThreads.listActiveWorktreeOwners(), + (owner) => + canonicalPathForComparison(owner.worktreePath).pipe( + Effect.map((worktreePath) => ({ owner, worktreePath })), + ), + { concurrency: "unbounded" }, ); + const activeOwner = activeOwners.find( + ({ owner, worktreePath }) => + owner.threadId !== job.threadId && worktreePath === normalizedWorktreePath, + )?.owner; if (activeOwner !== undefined) { const failedAt = yield* nowIso; yield* dispatchCleanup(job.threadId, { From e8c8d330ce8659e4870b3011ced2d0456f0b1c45 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:46:53 -0700 Subject: [PATCH 19/22] fix(sync): negotiate cleanup tombstones --- apps/server/src/cli/thread.ts | 4 +- .../Layers/ProjectionSnapshotQuery.test.ts | 8 ++- .../Layers/ProjectionSnapshotQuery.ts | 5 +- .../Services/ProjectionSnapshotQuery.ts | 7 +- apps/server/src/orchestration/http.ts | 5 +- apps/server/src/server.test.ts | 65 +++++++++++++++++++ apps/server/src/ws.ts | 50 +++++++++----- .../src/state/shell-sync.test.ts | 24 ++++++- packages/client-runtime/src/state/shell.ts | 28 ++++++-- .../src/state/shellSnapshotHttp.ts | 20 +++++- packages/contracts/src/environmentHttp.ts | 5 ++ packages/contracts/src/orchestration.ts | 2 + 12 files changed, 188 insertions(+), 35 deletions(-) diff --git a/apps/server/src/cli/thread.ts b/apps/server/src/cli/thread.ts index 8f1dc2026280..26a03c85135c 100644 --- a/apps/server/src/cli/thread.ts +++ b/apps/server/src/cli/thread.ts @@ -758,7 +758,7 @@ const tryRunLiveThreadRead = Effect.fn("tryRunLiveThreadRead")(function* ( const headers = { authorization: `Bearer ${token}` }; const sourceResult = yield* Effect.result( client.orchestration - .shellSnapshot({ headers }) + .shellSnapshot({ headers, payload: {} }) .pipe(Effect.timeout(THREAD_CLI_LIVE_TIMEOUT)), ); if (sourceResult._tag === "Failure") { @@ -888,7 +888,7 @@ const runThreadSend = Effect.fn("runThreadSend")(function* ( withSendSession(auth, (token) => Effect.gen(function* () { const headers = { authorization: `Bearer ${token}` }; - const shell = yield* client.orchestration.shellSnapshot({ headers }).pipe( + const shell = yield* client.orchestration.shellSnapshot({ headers, payload: {} }).pipe( Effect.timeout(THREAD_CLI_LIVE_TIMEOUT), Effect.mapError( (cause) => new ThreadCliError({ operation: "live send target lookup", cause }), diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 8217ed56776d..d20227cdc4d1 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -1644,7 +1644,13 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { WHERE thread_id = 'thread-deleted' `; - const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); + const legacyCleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); + assert.equal(legacyCleanupShellSnapshot.projects.length, 1); + assert.equal(legacyCleanupShellSnapshot.threads.length, 0); + + const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot({ + includeWorktreeCleanupTombstones: true, + }); assert.equal(cleanupShellSnapshot.projects.length, 1); assert.deepStrictEqual(cleanupShellSnapshot.threads[0]?.worktreeCleanup, { status: "deleting", diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index b1eafe6a1cc3..a5082756257b 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -1969,7 +1969,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }), ); - const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = () => + const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = (options = {}) => sql .withTransaction( Effect.all([ @@ -2059,7 +2059,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { : Result.failVoid, ), threads: Arr.filterMap(threadRows, (row) => - row.deletedAt === null || row.worktreeCleanup != null + row.deletedAt === null || + (options.includeWorktreeCleanupTombstones === true && row.worktreeCleanup != null) ? Result.succeed({ id: row.threadId, projectId: row.projectId, diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index 0a00253a2285..1b439c432d81 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -81,10 +81,9 @@ export interface ProjectionSnapshotQueryShape { * Returns only projects and thread shell summaries so clients can bootstrap * lightweight navigation state without hydrating every thread body. */ - readonly getShellSnapshot: () => Effect.Effect< - OrchestrationShellSnapshot, - ProjectionRepositoryError - >; + readonly getShellSnapshot: (options?: { + readonly includeWorktreeCleanupTombstones?: boolean; + }) => Effect.Effect<OrchestrationShellSnapshot, ProjectionRepositoryError>; /** * Read archived thread shell summaries for the archive page. diff --git a/apps/server/src/orchestration/http.ts b/apps/server/src/orchestration/http.ts index fddf3c8fcb4a..397078c5bd3b 100644 --- a/apps/server/src/orchestration/http.ts +++ b/apps/server/src/orchestration/http.ts @@ -90,7 +90,10 @@ export const orchestrationHttpApiLayer = HttpApiBuilder.group( yield* annotateEnvironmentRequest(args.endpoint.name); yield* requireEnvironmentScope(AuthOrchestrationReadScope); return yield* projectionSnapshotQuery - .getShellSnapshot() + .getShellSnapshot({ + includeWorktreeCleanupTombstones: + args.payload.includeWorktreeCleanupTombstones === "true", + }) .pipe( Effect.catch((cause) => failEnvironmentInternal("orchestration_snapshot_failed", cause), diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5803ed586328..79a99a757079 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -6893,6 +6893,71 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("hides cleanup tombstones from legacy shell subscribers", () => + Effect.gen(function* () { + const threadId = ThreadId.make("thread-cleanup-tombstone"); + const now = "2026-01-01T00:00:00.000Z"; + const event = { + sequence: 1, + eventId: EventId.make("event-cleanup-tombstone"), + aggregateKind: "thread", + aggregateId: threadId, + occurredAt: now, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + type: "thread.deleted", + payload: { threadId, deletedAt: now }, + } satisfies Extract<OrchestrationEvent, { type: "thread.deleted" }>; + const tombstone = makeDefaultOrchestrationThreadShell({ + id: threadId, + worktreeCleanup: { + status: "deleting", + repositoryRoot: "/repo", + worktreePath: "/repo-worktrees/thread-cleanup-tombstone", + startedAt: now, + }, + }); + + yield* buildAppUnderTest({ + layers: { + orchestrationEngine: { + latestSequence: Effect.succeed(1), + readEvents: () => Stream.make(event), + }, + projectionSnapshotQuery: { + getThreadShellById: () => Effect.succeed(Option.some(tombstone)), + }, + }, + }); + + const wsUrl = yield* getWsServerUrl("/ws"); + const legacyItem = yield* Effect.scoped( + withWsRpcClient(wsUrl, (client) => + client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 0 }).pipe( + Stream.runHead, + ), + ), + ); + const capableItem = yield* Effect.scoped( + withWsRpcClient(wsUrl, (client) => + client[ORCHESTRATION_WS_METHODS.subscribeShell]({ + afterSequence: 0, + includeWorktreeCleanupTombstones: true, + }).pipe(Stream.runHead), + ), + ); + + assert.deepEqual(Option.getOrThrow(legacyItem), { + kind: "thread-removed", + sequence: 1, + threadId, + }); + assert.equal(Option.getOrThrow(capableItem).kind, "thread-upserted"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("subscribeShell coalesces live bursts after the synchronization marker", () => Effect.gen(function* () { const busyThreadId = ThreadId.make("thread-live-busy"); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index ad753f5c4ad0..f237006f0981 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1299,6 +1299,20 @@ const makeWsRpcLayer = ( observeRpcStreamEffect( ORCHESTRATION_WS_METHODS.subscribeShell, Effect.gen(function* () { + const includeWorktreeCleanupTombstones = + input.includeWorktreeCleanupTombstones === true; + const adaptShellItemForClient = ( + item: OrchestrationShellStreamItem, + ): OrchestrationShellStreamItem => + !includeWorktreeCleanupTombstones && + item.kind === "thread-upserted" && + item.thread.worktreeCleanup != null + ? { + kind: "thread-removed", + sequence: item.sequence, + threadId: item.thread.id, + } + : item; // Coalesce the live shell stream per aggregate over a small window // so bursts of high-frequency events (streaming message deltas, // activity appends) collapse into a single shell refetch and never @@ -1321,18 +1335,22 @@ const makeWsRpcLayer = ( ); const bufferedLiveStream = coalesceShellLiveStream(Stream.fromQueue(liveBuffer)); - const loadSnapshot = projectionSnapshotQuery.getShellSnapshot().pipe( - Effect.tapError((cause) => - Effect.logError("orchestration shell snapshot load failed", { cause }), - ), - Effect.mapError( - (cause) => - new OrchestrationGetSnapshotError({ - message: "Failed to load orchestration shell snapshot", - cause, - }), - ), - ); + const loadSnapshot = projectionSnapshotQuery + .getShellSnapshot({ + includeWorktreeCleanupTombstones: includeWorktreeCleanupTombstones, + }) + .pipe( + Effect.tapError((cause) => + Effect.logError("orchestration shell snapshot load failed", { cause }), + ), + Effect.mapError( + (cause) => + new OrchestrationGetSnapshotError({ + message: "Failed to load orchestration shell snapshot", + cause, + }), + ), + ); // Offer the completion marker into the same queue as live events. // Anything buffered while snapshot/replay work was in flight is @@ -1371,7 +1389,7 @@ const makeWsRpcLayer = ( return Stream.concat( Stream.make({ kind: "snapshot" as const, snapshot }), synchronizedThenLive, - ); + ).pipe(Stream.map(adaptShellItemForClient)); } const catchUpStream = coalesceShellStream( // Replay only through the head captured above. Newer events @@ -1388,7 +1406,9 @@ const makeWsRpcLayer = ( }), ), ); - return Stream.concat(catchUpStream, synchronizedThenLive); + return Stream.concat(catchUpStream, synchronizedThenLive).pipe( + Stream.map(adaptShellItemForClient), + ); } const snapshot = yield* loadSnapshot; @@ -1398,7 +1418,7 @@ const makeWsRpcLayer = ( snapshot, }), synchronizedThenLive, - ); + ).pipe(Stream.map(adaptShellItemForClient)); }), { "rpc.aggregate": "orchestration" }, ), diff --git a/packages/client-runtime/src/state/shell-sync.test.ts b/packages/client-runtime/src/state/shell-sync.test.ts index 40e9bd80dc5b..9636719ec54e 100644 --- a/packages/client-runtime/src/state/shell-sync.test.ts +++ b/packages/client-runtime/src/state/shell-sync.test.ts @@ -50,7 +50,14 @@ const LIVE_SHELL_SNAPSHOT: OrchestrationShellSnapshot = { function session(client: WsRpcProtocolClient): RpcSession.RpcSession { return { client, - initialConfig: Effect.succeed({ shellResumeCompletionMarker: true } as never), + initialConfig: Effect.succeed({ + shellResumeCompletionMarker: true, + environment: { + capabilities: { + threadWorktreeCleanup: true, + }, + }, + } as never), ready: Effect.void, probe: Effect.void, closed: Effect.never, @@ -169,12 +176,15 @@ describe("environment shell synchronization", () => { const subscribeInputs = yield* Queue.unbounded<{ readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; + readonly includeWorktreeCleanupTombstones?: boolean; }>(); const loaderCalls = yield* Ref.make(0); + const loaderCapabilities = yield* Ref.make<ReadonlyArray<boolean | undefined>>([]); const client = { [ORCHESTRATION_WS_METHODS.subscribeShell]: (input: { readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; + readonly includeWorktreeCleanupTombstones?: boolean; }) => Stream.unwrap( Queue.offer(subscribeInputs, input).pipe(Effect.as(Stream.fromQueue(events))), @@ -208,7 +218,14 @@ describe("environment shell synchronization", () => { clear: () => Effect.void, }); const snapshotLoader = ShellSnapshotLoader.of({ - load: () => Ref.update(loaderCalls, (count) => count + 1).pipe(Effect.as(Option.none())), + load: (_prepared, options) => + Effect.all([ + Ref.update(loaderCalls, (count) => count + 1), + Ref.update(loaderCapabilities, (values) => [ + ...values, + options?.includeWorktreeCleanupTombstones, + ]), + ]).pipe(Effect.as(Option.none())), }); const shellState = yield* makeEnvironmentShellState().pipe( Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), @@ -223,7 +240,9 @@ describe("environment shell synchronization", () => { const subscribeInput = yield* Queue.take(subscribeInputs); expect(subscribeInput.afterSequence).toBeUndefined(); expect(subscribeInput.requestCompletionMarker).toBe(true); + expect(subscribeInput.includeWorktreeCleanupTombstones).toBe(true); expect(yield* Ref.get(loaderCalls)).toBe(1); + expect(yield* Ref.get(loaderCapabilities)).toEqual([true]); const synchronizing = yield* SubscriptionRef.get(shellState); expect(synchronizing.status).toBe("synchronizing"); expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(cachedSnapshot); @@ -243,6 +262,7 @@ describe("environment shell synchronization", () => { const resumedInput = yield* Queue.take(subscribeInputs); expect(resumedInput.afterSequence).toBe(resetSnapshot.snapshotSequence); expect(resumedInput.requestCompletionMarker).toBe(true); + expect(resumedInput.includeWorktreeCleanupTombstones).toBe(true); expect(yield* Ref.get(loaderCalls)).toBe(1); }), ); diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts index c150bbb75b8c..1f72ae549527 100644 --- a/packages/client-runtime/src/state/shell.ts +++ b/packages/client-runtime/src/state/shell.ts @@ -190,10 +190,18 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") ORCHESTRATION_WS_METHODS.subscribeShell, Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) { yield* Ref.set(activeSubscriptionSession, session); - const supportsCompletionMarker = yield* session.initialConfig.pipe( - Effect.map((config) => config.shellResumeCompletionMarker === true), - Effect.orElseSucceed(() => false), + const shellCapabilities = yield* session.initialConfig.pipe( + Effect.map((config) => ({ + completionMarker: config.shellResumeCompletionMarker === true, + worktreeCleanupTombstones: + config.environment.capabilities.threadWorktreeCleanup === true, + })), + Effect.orElseSucceed(() => ({ + completionMarker: false, + worktreeCleanupTombstones: false, + })), ); + const supportsCompletionMarker = shellCapabilities.completionMarker; yield* Ref.set(awaitingCompletion, supportsCompletionMarker); yield* setSynchronizing; @@ -218,7 +226,9 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") }), ), ); - const httpSnapshot = yield* snapshotLoader.load(prepared); + const httpSnapshot = yield* snapshotLoader.load(prepared, { + includeWorktreeCleanupTombstones: shellCapabilities.worktreeCleanupTombstones, + }); if (Option.isSome(httpSnapshot)) { yield* applyItem({ kind: "snapshot", snapshot: httpSnapshot.value }); canResume = true; @@ -229,7 +239,12 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") // If the authoritative refresh failed, omit the cached cursor so the // socket fallback sends a complete snapshot for this new session. if (!canResume || Option.isNone(current.snapshot)) { - return supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}; + return { + ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}), + ...(shellCapabilities.worktreeCleanupTombstones + ? { includeWorktreeCleanupTombstones: true as const } + : {}), + }; } if (!supportsCompletionMarker) { // Without a completion marker there is no synchronized signal for a @@ -243,6 +258,9 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") return { afterSequence: current.snapshot.value.snapshotSequence, ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}), + ...(shellCapabilities.worktreeCleanupTombstones + ? { includeWorktreeCleanupTombstones: true as const } + : {}), }; }), { diff --git a/packages/client-runtime/src/state/shellSnapshotHttp.ts b/packages/client-runtime/src/state/shellSnapshotHttp.ts index b0a492a1305f..8160b213974a 100644 --- a/packages/client-runtime/src/state/shellSnapshotHttp.ts +++ b/packages/client-runtime/src/state/shellSnapshotHttp.ts @@ -28,6 +28,7 @@ export const fetchEnvironmentShellSnapshot = Effect.fn( readonly prepared: PreparedConnection; readonly signer: Option.Option<ManagedRelayDpopSigner["Service"]>; readonly timeoutMs?: number; + readonly includeWorktreeCleanupTombstones?: boolean; }) { const requestUrl = environmentEndpointUrl(input.prepared.httpBaseUrl, "/api/orchestration/shell"); const client = yield* makeEnvironmentHttpApiClient(input.prepared.httpBaseUrl); @@ -42,7 +43,13 @@ export const fetchEnvironmentShellSnapshot = Effect.fn( input.timeoutMs ?? DEFAULT_SHELL_SNAPSHOT_TIMEOUT_MS, withEnvironmentCredentials( input.prepared.httpAuthorization, - client.orchestration.shellSnapshot({ headers }), + client.orchestration.shellSnapshot({ + payload: + input.includeWorktreeCleanupTombstones === true + ? { includeWorktreeCleanupTombstones: "true" as const } + : {}, + headers, + }), ), ); }); @@ -58,6 +65,7 @@ export class ShellSnapshotLoader extends Context.Service< { readonly load: ( prepared: PreparedConnection, + options?: { readonly includeWorktreeCleanupTombstones?: boolean }, ) => Effect.Effect<Option.Option<OrchestrationShellSnapshot>>; } >()("@t3tools/client-runtime/state/shellSnapshotHttp/ShellSnapshotLoader") {} @@ -74,8 +82,14 @@ export const shellSnapshotLoaderLayer: Layer.Layer< // connections, so the loader must not hard-require it. const signer = yield* Effect.serviceOption(ManagedRelayDpopSigner); return ShellSnapshotLoader.of({ - load: (prepared: PreparedConnection) => - fetchEnvironmentShellSnapshot({ prepared, signer }).pipe( + load: (prepared: PreparedConnection, options = {}) => + fetchEnvironmentShellSnapshot({ + prepared, + signer, + ...(options.includeWorktreeCleanupTombstones === true + ? { includeWorktreeCleanupTombstones: true } + : {}), + }).pipe( Effect.map(Option.some<OrchestrationShellSnapshot>), Effect.provideService(HttpClient.HttpClient, httpClient), Effect.catchCause((cause) => diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 8fef518d49bc..8f80e0b5ae83 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -508,6 +508,10 @@ const EnvironmentOrchestrationThreadSnapshotQuery = { beforeCursor: Schema.optional(TrimmedNonEmptyString), }; +const EnvironmentOrchestrationShellSnapshotQuery = { + includeWorktreeCleanupTombstones: Schema.optional(Schema.Literal("true")), +}; + export const ThreadWaitHandle = Schema.Struct({ kind: Schema.Literal("wait-handle"), environmentId: EnvironmentId, @@ -556,6 +560,7 @@ export class EnvironmentOrchestrationHttpApi extends HttpApiGroup.make("orchestr .add( HttpApiEndpoint.get("shellSnapshot", "/api/orchestration/shell", { headers: OptionalBearerHeaders, + payload: EnvironmentOrchestrationShellSnapshotQuery, success: OrchestrationShellSnapshot, error: EnvironmentOrchestrationSnapshotErrors, }).middleware(EnvironmentAuthenticatedAuth), diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index d99651cf1791..125ac22b3d2b 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -691,6 +691,8 @@ export const OrchestrationSubscribeShellInput = Schema.Struct({ * snapshot or catch-up replay and before it begins emitting live events. */ requestCompletionMarker: Schema.optionalKey(Schema.Boolean), + /** Include deleted thread shells that still need worktree cleanup or recovery. */ + includeWorktreeCleanupTombstones: Schema.optionalKey(Schema.Boolean), }); export type OrchestrationSubscribeShellInput = typeof OrchestrationSubscribeShellInput.Type; From 937f2b30b0038dec3fc750684c403970f3cfd9a6 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:46:53 -0700 Subject: [PATCH 20/22] fix(web): use tooltip width for cleanup paths --- apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx index 0cad74c5cf09..dbf259ee4031 100644 --- a/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx +++ b/apps/web/src/components/sidebar/SidebarThreadHoverContent.tsx @@ -140,7 +140,7 @@ export function SidebarThreadCleanupHoverContent(props: { {cleanup.status === "deleting" ? ( <> <div className="font-medium">Deleting worktree</div> - <div className="mt-1 break-all font-mono text-[10px] text-wrap opacity-80"> + <div className="mt-1 break-all font-mono text-[10px] text-left [text-wrap-style:auto] opacity-80"> {cleanup.worktreePath} </div> </> From a2c2cde67127123f82589efb8af9465c3acf4459 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 12:55:09 -0700 Subject: [PATCH 21/22] Revert "fix(sync): negotiate cleanup tombstones" This reverts commit e8c8d330ce8659e4870b3011ced2d0456f0b1c45. --- apps/server/src/cli/thread.ts | 4 +- .../Layers/ProjectionSnapshotQuery.test.ts | 8 +-- .../Layers/ProjectionSnapshotQuery.ts | 5 +- .../Services/ProjectionSnapshotQuery.ts | 7 +- apps/server/src/orchestration/http.ts | 5 +- apps/server/src/server.test.ts | 65 ------------------- apps/server/src/ws.ts | 50 +++++--------- .../src/state/shell-sync.test.ts | 24 +------ packages/client-runtime/src/state/shell.ts | 28 ++------ .../src/state/shellSnapshotHttp.ts | 20 +----- packages/contracts/src/environmentHttp.ts | 5 -- packages/contracts/src/orchestration.ts | 2 - 12 files changed, 35 insertions(+), 188 deletions(-) diff --git a/apps/server/src/cli/thread.ts b/apps/server/src/cli/thread.ts index 26a03c85135c..8f1dc2026280 100644 --- a/apps/server/src/cli/thread.ts +++ b/apps/server/src/cli/thread.ts @@ -758,7 +758,7 @@ const tryRunLiveThreadRead = Effect.fn("tryRunLiveThreadRead")(function* ( const headers = { authorization: `Bearer ${token}` }; const sourceResult = yield* Effect.result( client.orchestration - .shellSnapshot({ headers, payload: {} }) + .shellSnapshot({ headers }) .pipe(Effect.timeout(THREAD_CLI_LIVE_TIMEOUT)), ); if (sourceResult._tag === "Failure") { @@ -888,7 +888,7 @@ const runThreadSend = Effect.fn("runThreadSend")(function* ( withSendSession(auth, (token) => Effect.gen(function* () { const headers = { authorization: `Bearer ${token}` }; - const shell = yield* client.orchestration.shellSnapshot({ headers, payload: {} }).pipe( + const shell = yield* client.orchestration.shellSnapshot({ headers }).pipe( Effect.timeout(THREAD_CLI_LIVE_TIMEOUT), Effect.mapError( (cause) => new ThreadCliError({ operation: "live send target lookup", cause }), diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index d20227cdc4d1..8217ed56776d 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -1644,13 +1644,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { WHERE thread_id = 'thread-deleted' `; - const legacyCleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); - assert.equal(legacyCleanupShellSnapshot.projects.length, 1); - assert.equal(legacyCleanupShellSnapshot.threads.length, 0); - - const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot({ - includeWorktreeCleanupTombstones: true, - }); + const cleanupShellSnapshot = yield* snapshotQuery.getShellSnapshot(); assert.equal(cleanupShellSnapshot.projects.length, 1); assert.deepStrictEqual(cleanupShellSnapshot.threads[0]?.worktreeCleanup, { status: "deleting", diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index a5082756257b..b1eafe6a1cc3 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -1969,7 +1969,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { }), ); - const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = (options = {}) => + const getShellSnapshot: ProjectionSnapshotQueryShape["getShellSnapshot"] = () => sql .withTransaction( Effect.all([ @@ -2059,8 +2059,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { : Result.failVoid, ), threads: Arr.filterMap(threadRows, (row) => - row.deletedAt === null || - (options.includeWorktreeCleanupTombstones === true && row.worktreeCleanup != null) + row.deletedAt === null || row.worktreeCleanup != null ? Result.succeed({ id: row.threadId, projectId: row.projectId, diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts index 1b439c432d81..0a00253a2285 100644 --- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts @@ -81,9 +81,10 @@ export interface ProjectionSnapshotQueryShape { * Returns only projects and thread shell summaries so clients can bootstrap * lightweight navigation state without hydrating every thread body. */ - readonly getShellSnapshot: (options?: { - readonly includeWorktreeCleanupTombstones?: boolean; - }) => Effect.Effect<OrchestrationShellSnapshot, ProjectionRepositoryError>; + readonly getShellSnapshot: () => Effect.Effect< + OrchestrationShellSnapshot, + ProjectionRepositoryError + >; /** * Read archived thread shell summaries for the archive page. diff --git a/apps/server/src/orchestration/http.ts b/apps/server/src/orchestration/http.ts index 397078c5bd3b..fddf3c8fcb4a 100644 --- a/apps/server/src/orchestration/http.ts +++ b/apps/server/src/orchestration/http.ts @@ -90,10 +90,7 @@ export const orchestrationHttpApiLayer = HttpApiBuilder.group( yield* annotateEnvironmentRequest(args.endpoint.name); yield* requireEnvironmentScope(AuthOrchestrationReadScope); return yield* projectionSnapshotQuery - .getShellSnapshot({ - includeWorktreeCleanupTombstones: - args.payload.includeWorktreeCleanupTombstones === "true", - }) + .getShellSnapshot() .pipe( Effect.catch((cause) => failEnvironmentInternal("orchestration_snapshot_failed", cause), diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 79a99a757079..5803ed586328 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -6893,71 +6893,6 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); - it.effect("hides cleanup tombstones from legacy shell subscribers", () => - Effect.gen(function* () { - const threadId = ThreadId.make("thread-cleanup-tombstone"); - const now = "2026-01-01T00:00:00.000Z"; - const event = { - sequence: 1, - eventId: EventId.make("event-cleanup-tombstone"), - aggregateKind: "thread", - aggregateId: threadId, - occurredAt: now, - commandId: null, - causationEventId: null, - correlationId: null, - metadata: {}, - type: "thread.deleted", - payload: { threadId, deletedAt: now }, - } satisfies Extract<OrchestrationEvent, { type: "thread.deleted" }>; - const tombstone = makeDefaultOrchestrationThreadShell({ - id: threadId, - worktreeCleanup: { - status: "deleting", - repositoryRoot: "/repo", - worktreePath: "/repo-worktrees/thread-cleanup-tombstone", - startedAt: now, - }, - }); - - yield* buildAppUnderTest({ - layers: { - orchestrationEngine: { - latestSequence: Effect.succeed(1), - readEvents: () => Stream.make(event), - }, - projectionSnapshotQuery: { - getThreadShellById: () => Effect.succeed(Option.some(tombstone)), - }, - }, - }); - - const wsUrl = yield* getWsServerUrl("/ws"); - const legacyItem = yield* Effect.scoped( - withWsRpcClient(wsUrl, (client) => - client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 0 }).pipe( - Stream.runHead, - ), - ), - ); - const capableItem = yield* Effect.scoped( - withWsRpcClient(wsUrl, (client) => - client[ORCHESTRATION_WS_METHODS.subscribeShell]({ - afterSequence: 0, - includeWorktreeCleanupTombstones: true, - }).pipe(Stream.runHead), - ), - ); - - assert.deepEqual(Option.getOrThrow(legacyItem), { - kind: "thread-removed", - sequence: 1, - threadId, - }); - assert.equal(Option.getOrThrow(capableItem).kind, "thread-upserted"); - }).pipe(Effect.provide(NodeHttpServer.layerTest)), - ); - it.effect("subscribeShell coalesces live bursts after the synchronization marker", () => Effect.gen(function* () { const busyThreadId = ThreadId.make("thread-live-busy"); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f237006f0981..ad753f5c4ad0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1299,20 +1299,6 @@ const makeWsRpcLayer = ( observeRpcStreamEffect( ORCHESTRATION_WS_METHODS.subscribeShell, Effect.gen(function* () { - const includeWorktreeCleanupTombstones = - input.includeWorktreeCleanupTombstones === true; - const adaptShellItemForClient = ( - item: OrchestrationShellStreamItem, - ): OrchestrationShellStreamItem => - !includeWorktreeCleanupTombstones && - item.kind === "thread-upserted" && - item.thread.worktreeCleanup != null - ? { - kind: "thread-removed", - sequence: item.sequence, - threadId: item.thread.id, - } - : item; // Coalesce the live shell stream per aggregate over a small window // so bursts of high-frequency events (streaming message deltas, // activity appends) collapse into a single shell refetch and never @@ -1335,22 +1321,18 @@ const makeWsRpcLayer = ( ); const bufferedLiveStream = coalesceShellLiveStream(Stream.fromQueue(liveBuffer)); - const loadSnapshot = projectionSnapshotQuery - .getShellSnapshot({ - includeWorktreeCleanupTombstones: includeWorktreeCleanupTombstones, - }) - .pipe( - Effect.tapError((cause) => - Effect.logError("orchestration shell snapshot load failed", { cause }), - ), - Effect.mapError( - (cause) => - new OrchestrationGetSnapshotError({ - message: "Failed to load orchestration shell snapshot", - cause, - }), - ), - ); + const loadSnapshot = projectionSnapshotQuery.getShellSnapshot().pipe( + Effect.tapError((cause) => + Effect.logError("orchestration shell snapshot load failed", { cause }), + ), + Effect.mapError( + (cause) => + new OrchestrationGetSnapshotError({ + message: "Failed to load orchestration shell snapshot", + cause, + }), + ), + ); // Offer the completion marker into the same queue as live events. // Anything buffered while snapshot/replay work was in flight is @@ -1389,7 +1371,7 @@ const makeWsRpcLayer = ( return Stream.concat( Stream.make({ kind: "snapshot" as const, snapshot }), synchronizedThenLive, - ).pipe(Stream.map(adaptShellItemForClient)); + ); } const catchUpStream = coalesceShellStream( // Replay only through the head captured above. Newer events @@ -1406,9 +1388,7 @@ const makeWsRpcLayer = ( }), ), ); - return Stream.concat(catchUpStream, synchronizedThenLive).pipe( - Stream.map(adaptShellItemForClient), - ); + return Stream.concat(catchUpStream, synchronizedThenLive); } const snapshot = yield* loadSnapshot; @@ -1418,7 +1398,7 @@ const makeWsRpcLayer = ( snapshot, }), synchronizedThenLive, - ).pipe(Stream.map(adaptShellItemForClient)); + ); }), { "rpc.aggregate": "orchestration" }, ), diff --git a/packages/client-runtime/src/state/shell-sync.test.ts b/packages/client-runtime/src/state/shell-sync.test.ts index 9636719ec54e..40e9bd80dc5b 100644 --- a/packages/client-runtime/src/state/shell-sync.test.ts +++ b/packages/client-runtime/src/state/shell-sync.test.ts @@ -50,14 +50,7 @@ const LIVE_SHELL_SNAPSHOT: OrchestrationShellSnapshot = { function session(client: WsRpcProtocolClient): RpcSession.RpcSession { return { client, - initialConfig: Effect.succeed({ - shellResumeCompletionMarker: true, - environment: { - capabilities: { - threadWorktreeCleanup: true, - }, - }, - } as never), + initialConfig: Effect.succeed({ shellResumeCompletionMarker: true } as never), ready: Effect.void, probe: Effect.void, closed: Effect.never, @@ -176,15 +169,12 @@ describe("environment shell synchronization", () => { const subscribeInputs = yield* Queue.unbounded<{ readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; - readonly includeWorktreeCleanupTombstones?: boolean; }>(); const loaderCalls = yield* Ref.make(0); - const loaderCapabilities = yield* Ref.make<ReadonlyArray<boolean | undefined>>([]); const client = { [ORCHESTRATION_WS_METHODS.subscribeShell]: (input: { readonly afterSequence?: number; readonly requestCompletionMarker?: boolean; - readonly includeWorktreeCleanupTombstones?: boolean; }) => Stream.unwrap( Queue.offer(subscribeInputs, input).pipe(Effect.as(Stream.fromQueue(events))), @@ -218,14 +208,7 @@ describe("environment shell synchronization", () => { clear: () => Effect.void, }); const snapshotLoader = ShellSnapshotLoader.of({ - load: (_prepared, options) => - Effect.all([ - Ref.update(loaderCalls, (count) => count + 1), - Ref.update(loaderCapabilities, (values) => [ - ...values, - options?.includeWorktreeCleanupTombstones, - ]), - ]).pipe(Effect.as(Option.none())), + load: () => Ref.update(loaderCalls, (count) => count + 1).pipe(Effect.as(Option.none())), }); const shellState = yield* makeEnvironmentShellState().pipe( Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor), @@ -240,9 +223,7 @@ describe("environment shell synchronization", () => { const subscribeInput = yield* Queue.take(subscribeInputs); expect(subscribeInput.afterSequence).toBeUndefined(); expect(subscribeInput.requestCompletionMarker).toBe(true); - expect(subscribeInput.includeWorktreeCleanupTombstones).toBe(true); expect(yield* Ref.get(loaderCalls)).toBe(1); - expect(yield* Ref.get(loaderCapabilities)).toEqual([true]); const synchronizing = yield* SubscriptionRef.get(shellState); expect(synchronizing.status).toBe("synchronizing"); expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(cachedSnapshot); @@ -262,7 +243,6 @@ describe("environment shell synchronization", () => { const resumedInput = yield* Queue.take(subscribeInputs); expect(resumedInput.afterSequence).toBe(resetSnapshot.snapshotSequence); expect(resumedInput.requestCompletionMarker).toBe(true); - expect(resumedInput.includeWorktreeCleanupTombstones).toBe(true); expect(yield* Ref.get(loaderCalls)).toBe(1); }), ); diff --git a/packages/client-runtime/src/state/shell.ts b/packages/client-runtime/src/state/shell.ts index 1f72ae549527..c150bbb75b8c 100644 --- a/packages/client-runtime/src/state/shell.ts +++ b/packages/client-runtime/src/state/shell.ts @@ -190,18 +190,10 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") ORCHESTRATION_WS_METHODS.subscribeShell, Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) { yield* Ref.set(activeSubscriptionSession, session); - const shellCapabilities = yield* session.initialConfig.pipe( - Effect.map((config) => ({ - completionMarker: config.shellResumeCompletionMarker === true, - worktreeCleanupTombstones: - config.environment.capabilities.threadWorktreeCleanup === true, - })), - Effect.orElseSucceed(() => ({ - completionMarker: false, - worktreeCleanupTombstones: false, - })), + const supportsCompletionMarker = yield* session.initialConfig.pipe( + Effect.map((config) => config.shellResumeCompletionMarker === true), + Effect.orElseSucceed(() => false), ); - const supportsCompletionMarker = shellCapabilities.completionMarker; yield* Ref.set(awaitingCompletion, supportsCompletionMarker); yield* setSynchronizing; @@ -226,9 +218,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") }), ), ); - const httpSnapshot = yield* snapshotLoader.load(prepared, { - includeWorktreeCleanupTombstones: shellCapabilities.worktreeCleanupTombstones, - }); + const httpSnapshot = yield* snapshotLoader.load(prepared); if (Option.isSome(httpSnapshot)) { yield* applyItem({ kind: "snapshot", snapshot: httpSnapshot.value }); canResume = true; @@ -239,12 +229,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") // If the authoritative refresh failed, omit the cached cursor so the // socket fallback sends a complete snapshot for this new session. if (!canResume || Option.isNone(current.snapshot)) { - return { - ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}), - ...(shellCapabilities.worktreeCleanupTombstones - ? { includeWorktreeCleanupTombstones: true as const } - : {}), - }; + return supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}; } if (!supportsCompletionMarker) { // Without a completion marker there is no synchronized signal for a @@ -258,9 +243,6 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make") return { afterSequence: current.snapshot.value.snapshotSequence, ...(supportsCompletionMarker ? { requestCompletionMarker: true as const } : {}), - ...(shellCapabilities.worktreeCleanupTombstones - ? { includeWorktreeCleanupTombstones: true as const } - : {}), }; }), { diff --git a/packages/client-runtime/src/state/shellSnapshotHttp.ts b/packages/client-runtime/src/state/shellSnapshotHttp.ts index 8160b213974a..b0a492a1305f 100644 --- a/packages/client-runtime/src/state/shellSnapshotHttp.ts +++ b/packages/client-runtime/src/state/shellSnapshotHttp.ts @@ -28,7 +28,6 @@ export const fetchEnvironmentShellSnapshot = Effect.fn( readonly prepared: PreparedConnection; readonly signer: Option.Option<ManagedRelayDpopSigner["Service"]>; readonly timeoutMs?: number; - readonly includeWorktreeCleanupTombstones?: boolean; }) { const requestUrl = environmentEndpointUrl(input.prepared.httpBaseUrl, "/api/orchestration/shell"); const client = yield* makeEnvironmentHttpApiClient(input.prepared.httpBaseUrl); @@ -43,13 +42,7 @@ export const fetchEnvironmentShellSnapshot = Effect.fn( input.timeoutMs ?? DEFAULT_SHELL_SNAPSHOT_TIMEOUT_MS, withEnvironmentCredentials( input.prepared.httpAuthorization, - client.orchestration.shellSnapshot({ - payload: - input.includeWorktreeCleanupTombstones === true - ? { includeWorktreeCleanupTombstones: "true" as const } - : {}, - headers, - }), + client.orchestration.shellSnapshot({ headers }), ), ); }); @@ -65,7 +58,6 @@ export class ShellSnapshotLoader extends Context.Service< { readonly load: ( prepared: PreparedConnection, - options?: { readonly includeWorktreeCleanupTombstones?: boolean }, ) => Effect.Effect<Option.Option<OrchestrationShellSnapshot>>; } >()("@t3tools/client-runtime/state/shellSnapshotHttp/ShellSnapshotLoader") {} @@ -82,14 +74,8 @@ export const shellSnapshotLoaderLayer: Layer.Layer< // connections, so the loader must not hard-require it. const signer = yield* Effect.serviceOption(ManagedRelayDpopSigner); return ShellSnapshotLoader.of({ - load: (prepared: PreparedConnection, options = {}) => - fetchEnvironmentShellSnapshot({ - prepared, - signer, - ...(options.includeWorktreeCleanupTombstones === true - ? { includeWorktreeCleanupTombstones: true } - : {}), - }).pipe( + load: (prepared: PreparedConnection) => + fetchEnvironmentShellSnapshot({ prepared, signer }).pipe( Effect.map(Option.some<OrchestrationShellSnapshot>), Effect.provideService(HttpClient.HttpClient, httpClient), Effect.catchCause((cause) => diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 8f80e0b5ae83..8fef518d49bc 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -508,10 +508,6 @@ const EnvironmentOrchestrationThreadSnapshotQuery = { beforeCursor: Schema.optional(TrimmedNonEmptyString), }; -const EnvironmentOrchestrationShellSnapshotQuery = { - includeWorktreeCleanupTombstones: Schema.optional(Schema.Literal("true")), -}; - export const ThreadWaitHandle = Schema.Struct({ kind: Schema.Literal("wait-handle"), environmentId: EnvironmentId, @@ -560,7 +556,6 @@ export class EnvironmentOrchestrationHttpApi extends HttpApiGroup.make("orchestr .add( HttpApiEndpoint.get("shellSnapshot", "/api/orchestration/shell", { headers: OptionalBearerHeaders, - payload: EnvironmentOrchestrationShellSnapshotQuery, success: OrchestrationShellSnapshot, error: EnvironmentOrchestrationSnapshotErrors, }).middleware(EnvironmentAuthenticatedAuth), diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 125ac22b3d2b..d99651cf1791 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -691,8 +691,6 @@ export const OrchestrationSubscribeShellInput = Schema.Struct({ * snapshot or catch-up replay and before it begins emitting live events. */ requestCompletionMarker: Schema.optionalKey(Schema.Boolean), - /** Include deleted thread shells that still need worktree cleanup or recovery. */ - includeWorktreeCleanupTombstones: Schema.optionalKey(Schema.Boolean), }); export type OrchestrationSubscribeShellInput = typeof OrchestrationSubscribeShellInput.Type; From 2803d893de8b01f038b33055e3f0e99445b68ee7 Mon Sep 17 00:00:00 2001 From: Michael Johnston <lastobelus@mac.com> Date: Mon, 24 Aug 2026 13:13:13 -0700 Subject: [PATCH 22/22] fix(server): close cleanup concurrency gaps --- .../Layers/ThreadDeletionReactor.test.ts | 19 +++-- .../Layers/ThreadDeletionReactor.ts | 70 +++++++++++-------- .../src/orchestration/decider.delete.test.ts | 32 +++++++++ apps/server/src/orchestration/decider.ts | 14 ++++ apps/server/src/vcs/GitVcsDriverCore.test.ts | 44 ++++++++++++ apps/server/src/vcs/GitVcsDriverCore.ts | 23 ++++++ packages/contracts/src/git.ts | 1 + 7 files changed, 170 insertions(+), 33 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts index 6eb90d2123e9..3f403a5f945e 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts @@ -189,6 +189,8 @@ describe("durable worktree cleanup", () => { }; const rows = new Map([[thread.threadId, thread]]); const operations: string[] = []; + const teardownStarted = yield* Deferred.make<void, never>(); + const releaseTeardown = yield* Deferred.make<void, never>(); const removed = yield* Deferred.make<void>(); const completionDispatchFailed = yield* Deferred.make<void>(); let completionDispatchAttempts = 0; @@ -237,7 +239,10 @@ describe("durable worktree cleanup", () => { }), Layer.mock(ProviderService)({ stopSession: ({ threadId }) => - Effect.sync(() => void operations.push(`stop:${threadId}`)), + Effect.sync(() => void operations.push(`stop:${threadId}`)).pipe( + Effect.andThen(Deferred.succeed(teardownStarted, undefined)), + Effect.andThen(Deferred.await(releaseTeardown)), + ), }), Layer.mock(TerminalManager.TerminalManager)({ close: ({ threadId }) => Effect.sync(() => void operations.push(`close:${threadId}`)), @@ -253,8 +258,14 @@ describe("durable worktree cleanup", () => { yield* Effect.gen(function* () { const reactor = yield* ThreadDeletionReactor; yield* reactor.start(); + yield* Deferred.await(teardownStarted); + const drainCompleted = yield* Deferred.make<void, never>(); + const drain = yield* Effect.forkChild( + reactor.drain.pipe(Effect.andThen(Deferred.succeed(drainCompleted, undefined))), + ); + expect(yield* Deferred.isDone(drainCompleted)).toBe(false); + yield* Deferred.succeed(releaseTeardown, undefined); yield* Deferred.await(removed); - const drain = yield* Effect.forkChild(reactor.drain); yield* Deferred.await(completionDispatchFailed); yield* TestClock.adjust("1 second"); yield* Fiber.join(drain); @@ -649,7 +660,7 @@ describe("durable worktree cleanup", () => { }), }), Layer.mock(GitWorkflowService)({ - removeWorktree: ({ path }) => + removeWorktree: ({ path, allowMissing }) => Effect.gen(function* () { removals.push(path); operations.push(`remove:${path}`); @@ -661,7 +672,7 @@ describe("durable worktree cleanup", () => { detail: "permission denied", }); } - if (path === fourth.worktreePath) { + if (path === fourth.worktreePath && allowMissing !== true) { return yield* new GitCommandError({ operation: "remove worktree", command: "git worktree remove", diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 3ea6809431b7..ee23efa5d171 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -200,8 +200,6 @@ const make = Effect.gen(function* () { ); }; - const worker = yield* makeDrainableWorker(processThreadDeletedSafely); - const processCleanup = Effect.fn("processThreadWorktreeCleanup")(function* (job: CleanupJob) { if (job.needsTeardown) { yield* stopProviderSessionStrict(job.threadId); @@ -277,15 +275,10 @@ const make = Effect.gen(function* () { cwd: deleting.repositoryRoot, path: deleting.worktreePath, force: true, + allowMissing: true, }), ); - // A restart can observe `deleting` after Git removed the worktree but - // before the completion event was persisted. Git quite reasonably rejects - // a second removal of an unregistered path, so an absent path is already - // the desired end state and should complete the durable cleanup. - const alreadyRemoved = - Result.isFailure(removal) && !(yield* fileSystem.exists(deleting.worktreePath)); - if (Result.isSuccess(removal) || alreadyRemoved) { + if (Result.isSuccess(removal)) { yield* dispatchCleanupWithRetry(job.threadId, null); return; } @@ -448,31 +441,50 @@ const make = Effect.gen(function* () { return Effect.void; }; - const cleanupDrain = Effect.gen(function* () { - const workers = yield* Ref.get(cleanupWorkersRef); - yield* Effect.forEach(workers.values(), (entry) => entry.worker.drain, { - concurrency: "unbounded", - }); + const worker = yield* makeDrainableWorker((event: ThreadDeletedEvent) => + processThreadDeletedSafely(event).pipe( + Effect.andThen(Ref.get(failedThreadTeardownIdsRef)), + Effect.flatMap((failedThreadIds) => + failedThreadIds.has(event.payload.threadId) ? Effect.void : enqueueCleanupFromEvent(event), + ), + ), + ); + + const cleanupDrain: Effect.Effect<void> = Effect.gen(function* () { + while (true) { + const snapshot = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const workers = yield* Ref.get(cleanupWorkersRef); + return yield* Effect.forEach(Array.from(workers.entries()), ([repositoryKey, entry]) => + Ref.get(entry.generation).pipe( + Effect.map((generation) => ({ repositoryKey, entry, generation })), + ), + ); + }), + ); + yield* Effect.forEach(snapshot, ({ entry }) => entry.worker.drain, { + concurrency: "unbounded", + }); + const stable = yield* cleanupWorkersMutex.withPermit( + Effect.gen(function* () { + const current = yield* Ref.get(cleanupWorkersRef); + if (current.size !== snapshot.length) return false; + const checks = yield* Effect.forEach(snapshot, ({ repositoryKey, entry, generation }) => { + if (current.get(repositoryKey) !== entry) return Effect.succeed(false); + return Ref.get(entry.generation).pipe(Effect.map((value) => value === generation)); + }); + return checks.every(Boolean); + }), + ); + if (stable) return; + } }); const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () { yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if (event.type === "thread.deleted") { - return worker - .enqueue(event) - .pipe( - Effect.andThen(worker.drain), - Effect.andThen( - Ref.get(failedThreadTeardownIdsRef).pipe( - Effect.flatMap((failedThreadIds) => - failedThreadIds.has(event.payload.threadId) - ? Effect.void - : enqueueCleanupFromEvent(event), - ), - ), - ), - ); + return worker.enqueue(event); } return enqueueCleanupFromEvent(event); }), @@ -497,7 +509,7 @@ const make = Effect.gen(function* () { return { start, - drain: Effect.all([worker.drain, cleanupDrain]).pipe(Effect.asVoid), + drain: worker.drain.pipe(Effect.andThen(cleanupDrain)), } satisfies ThreadDeletionReactorShape; }); diff --git a/apps/server/src/orchestration/decider.delete.test.ts b/apps/server/src/orchestration/decider.delete.test.ts index eaec21c218f3..e7b1f87b7248 100644 --- a/apps/server/src/orchestration/decider.delete.test.ts +++ b/apps/server/src/orchestration/decider.delete.test.ts @@ -389,6 +389,38 @@ it.layer(NodeServices.layer)("decider deletion flows", (it) => { ); expect(pathReuseError.message).toContain("is still being cleaned up by thread"); + const projectCreateReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.create", + commandId: asCommandId("cmd-cleanup-project-create-reuse"), + projectId: asProjectId("project-cleanup-reuse"), + title: "Cleanup reuse", + workspaceRoot: "/tmp/project-delete-worktrees/cleanup-retry", + createdAt: "2026-01-01T00:00:00.000Z", + }, + readModel: afterDelete, + }), + ); + expect(projectCreateReuseError.message).toContain( + "is still being cleaned up by thread 'thread-delete-1'", + ); + + const projectUpdateReuseError = yield* Effect.flip( + decideOrchestrationCommand({ + command: { + type: "project.meta.update", + commandId: asCommandId("cmd-cleanup-project-update-reuse"), + projectId: asProjectId("project-delete"), + workspaceRoot: "/tmp/project-delete-worktrees/cleanup-retry", + }, + readModel: afterDelete, + }), + ); + expect(projectUpdateReuseError.message).toContain( + "is still being cleaned up by thread 'thread-delete-1'", + ); + const failed = yield* decideOrchestrationCommand({ command: { type: "thread.worktree-cleanup.update", diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 2e220e091a44..aeb3cebddddf 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -309,6 +309,13 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" workspaceRoot: command.workspaceRoot, exceptProjectId: command.projectId, }); + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.workspaceRoot); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Workspace root '${command.workspaceRoot}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } return { ...(yield* withEventBase({ @@ -344,6 +351,13 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" workspaceRoot: command.workspaceRoot, exceptProjectId: command.projectId, }); + const cleanupOwner = findWorktreeCleanupOwner(readModel, command.workspaceRoot); + if (cleanupOwner !== undefined) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: `Workspace root '${command.workspaceRoot}' is still being cleaned up by thread '${cleanupOwner.id}'.`, + }); + } } const occurredAt = yield* nowIso; return { diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index d6688d369692..562722b156c6 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -758,6 +758,50 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => { assert.notInclude(error.detail, "Git command failed in"); }), ); + + it.effect("allows an unregistered missing worktree when explicitly requested", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const pathService = yield* Path.Path; + const missingWorktree = pathService.join(cwd, "missing-worktree"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.initRepo({ cwd }); + + yield* driver.removeWorktree({ + cwd, + path: missingWorktree, + force: true, + allowMissing: true, + }); + }), + ); + + it.effect("does not treat an absent locked worktree as removed", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const worktreePath = pathService.join(yield* makeTmpDir("git-worktrees-"), "locked"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/locked-worktree", + }); + yield* git(cwd, ["worktree", "lock", worktreePath]); + yield* fileSystem.remove(worktreePath, { recursive: true }); + + const error = yield* driver + .removeWorktree({ cwd, path: worktreePath, force: true, allowMissing: true }) + .pipe(Effect.flip); + const registered = yield* git(cwd, ["worktree", "list", "--porcelain"]); + + assert.equal(error._tag, "GitCommandError"); + assert.include(registered, worktreePath); + }), + ); }); describe("review diff previews", () => { diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 97e3f8dc9265..7119174a40b3 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -2991,6 +2991,29 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* args.push("--force"); } args.push(input.path); + if (input.allowMissing === true) { + const result = yield* executeGitWithStableDiagnostics( + "GitVcsDriver.removeWorktree", + input.cwd, + args, + { + allowNonZeroExit: true, + timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, + }, + ); + if (result.exitCode === 0 || result.stderr.includes("is not a working tree")) return; + return yield* new GitCommandError({ + ...gitCommandContext({ + operation: "GitVcsDriver.removeWorktree", + cwd: input.cwd, + args, + }), + detail: "git worktree remove failed", + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }); + } yield* executeGit("GitVcsDriver.removeWorktree", input.cwd, args, { timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS, fallbackErrorDetail: "git worktree remove failed", diff --git a/packages/contracts/src/git.ts b/packages/contracts/src/git.ts index 915c3627c9b9..011d3453c3c8 100644 --- a/packages/contracts/src/git.ts +++ b/packages/contracts/src/git.ts @@ -161,6 +161,7 @@ export const VcsRemoveWorktreeInput = Schema.Struct({ cwd: TrimmedNonEmptyStringSchema, path: TrimmedNonEmptyStringSchema, force: Schema.optional(Schema.Boolean), + allowMissing: Schema.optional(Schema.Boolean), }); export type VcsRemoveWorktreeInput = typeof VcsRemoveWorktreeInput.Type;