diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 4e3c53268f24..79ea682e6992 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -53,6 +53,7 @@ import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; import { GitConfirmSheet } from "./features/threads/git/GitConfirmSheet"; import { GitOverviewSheet } from "./features/threads/git/GitOverviewSheet"; import { ThreadAgentsSheet } from "./features/threads/ThreadAgentsSheet"; +import { TaskGraphSheet } from "./features/threads/TaskGraphSheet"; import { ThreadQueueSheet } from "./features/threads/ThreadQueueControl"; import { ThreadRouteScreen } from "./features/threads/ThreadRouteScreen"; import { McpAppFullscreenScreen } from "./features/threads/McpAppFullscreenScreen"; @@ -564,6 +565,7 @@ const WORKSPACE_OVERLAY_ROUTES = new Set([ "SettingsSheet", "ThreadAgents", "ThreadQueue", + "ThreadTaskGraphs", "ThreadReviewComment", "ThreadDevicePreview", "ThreadBrowserPreview", @@ -818,6 +820,15 @@ const RootStackConfig = createWorkspaceStackNavigator({ sheetGrabberVisible: true, }, }), + ThreadTaskGraphs: createNativeStackScreen({ + screen: TaskGraphSheet, + options: { + ...FORM_SHEET_PRESENTATION_OPTIONS, + headerShown: false, + sheetAllowedDetents: [0.6, 0.95], + sheetGrabberVisible: true, + }, + }), GitOverview: createNativeStackScreen({ screen: GitOverviewSheet, linking: `${THREAD_LINKING_PREFIX}/git`, diff --git a/apps/mobile/src/features/threads/TaskGraphSheet.tsx b/apps/mobile/src/features/threads/TaskGraphSheet.tsx new file mode 100644 index 000000000000..93e6de45462c --- /dev/null +++ b/apps/mobile/src/features/threads/TaskGraphSheet.tsx @@ -0,0 +1,456 @@ +import { useAtomValue } from "@effect/atom-react"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, + type AtomCommandResult, +} from "@t3tools/client-runtime/state/runtime"; +import type { + EnvironmentId, + ModelSelection, + TaskGraph, + TaskGraphEdit, + TaskGraphNode, + ThreadId, +} from "@t3tools/contracts"; +import { formatModelSlugName } from "@t3tools/shared/model"; +import { taskGraphLayers } from "@t3tools/shared/taskGraph"; +import { StackActions, useNavigation, type StaticScreenProps } from "@react-navigation/native"; +import * as Haptics from "expo-haptics"; +import { useState } from "react"; +import { Alert, Platform, Pressable, ScrollView, View } from "react-native"; +import { Screen, ScreenStack, ScreenStackHeaderConfig } from "react-native-screens"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { AndroidSheetHeader } from "../../components/AndroidScreenHeader"; +import { AppText as Text } from "../../components/AppText"; +import { showConfirmDialog } from "../../components/ConfirmDialogHost"; +import { tryOpenExternalUrl } from "../../lib/openExternalUrl"; +import { useUniwindTheme } from "../../lib/useUniwindTheme"; +import { nativeHeaderScrollEdgeEffects } from "../../native/StackHeader"; +import { useEnvironmentPresentation } from "../../state/presentation"; +import { useEnvironmentQuery } from "../../state/query"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; +import { RequestActionButton } from "./RequestActionButton"; +import { SUBAGENT_TONE_TEXT_CLASS, SubagentStatusDot } from "./SubagentStatusDot"; +import { + sortTaskGraphs, + taskGraphActions, + taskGraphNodeActions, + taskGraphNodeStatusLabel, + taskGraphNodeTone, + taskGraphNodeWaitDetail, + taskGraphSummaryLabel, + taskGraphTone, +} from "./task-graph-presentation"; + +const HEADER_SCROLL_EDGE_EFFECTS = nativeHeaderScrollEdgeEffects(Platform.OS, Platform.Version); + +type TaskGraphTarget = { readonly environmentId: EnvironmentId; readonly threadId: ThreadId }; + +const useTaskGraphsQuery = (target: TaskGraphTarget) => + useEnvironmentQuery( + serverEnvironment.taskGraphsLive({ + environmentId: target.environmentId, + input: { threadId: target.threadId }, + }), + ); + +/** The thread's task graphs, live. Null until the first snapshot arrives. */ +export function useThreadTaskGraphs(target: TaskGraphTarget): ReadonlyArray | null { + return useTaskGraphsQuery(target).data?.graphs ?? null; +} + +function confirm(input: { + readonly title: string; + readonly message: string; + readonly confirmText: string; + readonly onConfirm: () => void; +}) { + if (Platform.OS === "ios") { + Alert.alert(input.title, input.message, [ + { text: "Keep", style: "cancel" }, + { text: input.confirmText, style: "destructive", onPress: input.onConfirm }, + ]); + return; + } + showConfirmDialog({ ...input, cancelText: "Keep", destructive: true }); +} + +/** + * Read-only view of a thread's task graphs, with the actions that move a + * graph between states: run a draft, cancel, cancel a branch, retry a node. + * Editing the graph itself happens on web and desktop. + */ +export function TaskGraphSheet({ route }: StaticScreenProps) { + const target = route.params; + const navigation = useNavigation(); + const insets = useSafeAreaInsets(); + const theme = useUniwindTheme(); + const query = useTaskGraphsQuery(target); + const graphs = query.data === null ? null : sortTaskGraphs(query.data.graphs); + const canEdit = useAtomValue( + serverEnvironment.editTaskGraph.permissionAtom(target.environmentId), + ); + const canRun = useAtomValue(serverEnvironment.runTaskGraph.permissionAtom(target.environmentId)); + const canCancel = useAtomValue( + serverEnvironment.cancelTaskGraph.permissionAtom(target.environmentId), + ); + const editGraph = useAtomCommand(serverEnvironment.editTaskGraph, { + label: "task graph edit", + reportFailure: false, + }); + const runGraph = useAtomCommand(serverEnvironment.runTaskGraph, { + label: "task graph run", + reportFailure: false, + }); + const cancelGraph = useAtomCommand(serverEnvironment.cancelTaskGraph, { + label: "task graph cancel", + reportFailure: false, + }); + const providers = useAtomValue(serverEnvironment.providersValueAtom(target.environmentId)); + const environmentLabel = + useEnvironmentPresentation(target.environmentId).presentation?.entry.target.label ?? + "This machine"; + // Servers without peer support reject this; the graph's own machine still has a name. + const peers = + useEnvironmentQuery( + serverEnvironment.taskGraphPeersLive({ environmentId: target.environmentId, input: {} }), + ).data?.peers ?? []; + const modelLabel = (selection: ModelSelection) => + providers + ?.find((provider) => provider.instanceId === selection.instanceId) + ?.models.find((model) => model.slug === selection.model)?.name ?? + formatModelSlugName(selection.model); + /** "Claude Opus 5.5 · on build-box": the node's model and where it ran or is pinned. */ + const placementLabel = (graph: TaskGraph, node: TaskGraphNode) => { + const model = node.modelSelection ?? graph.modelSelection; + // A node continuing its dependency's worktree goes where that one ran, known once it starts. + const machine = + node.assignedEnvironmentId ?? (node.workspace === "dependency" ? null : node.environmentId); + const machineLabel = + machine === null + ? null + : machine === target.environmentId + ? environmentLabel + : (peers.find((peer) => peer.environmentId === machine)?.label ?? "another machine"); + return [ + model === null ? null : modelLabel(model), + machineLabel === null ? null : `on ${machineLabel}`, + ] + .filter((part) => part !== null) + .join(" · "); + }; + // Keys of actions in flight, so a second tap cannot send the same edit twice. + const [pending, setPending] = useState>(() => new Set()); + + const perform = async ( + key: string, + failureTitle: string, + action: () => Promise>, + ) => { + setPending((current) => new Set(current).add(key)); + try { + const result = await action(); + if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) { + Alert.alert(failureTitle, String(squashAtomCommandFailure(result))); + } + } finally { + setPending((current) => { + const next = new Set(current); + next.delete(key); + return next; + }); + } + }; + + const edit = (graph: TaskGraph, change: TaskGraphEdit & { readonly key: string }) => + perform( + `${graph.id}:${change.key}`, + change.type === "retry_node" ? "Could not retry task" : "Could not cancel branch", + () => + editGraph({ + environmentId: target.environmentId, + input: { graphId: graph.id, edits: [change] }, + }), + ); + + const onRun = (graph: TaskGraph) => { + void Haptics.selectionAsync(); + void perform(graph.id, "Could not run graph", () => + runGraph({ environmentId: target.environmentId, input: { graphId: graph.id } }), + ); + }; + + const onCancel = (graph: TaskGraph) => + confirm({ + title: "Cancel task graph?", + message: `Every unfinished task in “${graph.title}” stops. Finished tasks and their pull requests stay.`, + confirmText: "Cancel graph", + onConfirm: () => + void perform(graph.id, "Could not cancel graph", () => + cancelGraph({ environmentId: target.environmentId, input: { graphId: graph.id } }), + ), + }); + + const onCancelBranch = (graph: TaskGraph, node: TaskGraphNode) => + confirm({ + title: "Cancel this branch?", + message: `“${node.title}” and every task that depends on it stop.`, + confirmText: "Cancel branch", + onConfirm: () => void edit(graph, { type: "cancel_branch", key: node.key }), + }); + + const onRetry = (graph: TaskGraph, node: TaskGraphNode) => { + void Haptics.selectionAsync(); + void edit(graph, { type: "retry_node", key: node.key }); + }; + + const openNodeThread = (node: TaskGraphNode) => { + if (node.threadId === null) return; + void Haptics.selectionAsync(); + // Replace rather than push, like the agents sheet: the node thread belongs + // in the workspace stack, not on top of a dismissed sheet. + navigation.dispatch( + StackActions.replace("Thread", { + environmentId: node.assignedEnvironmentId ?? target.environmentId, + threadId: node.threadId, + }), + ); + }; + + const openPullRequest = async (url: string) => { + if (!(await tryOpenExternalUrl(url, "pull-request"))) { + Alert.alert("Unable to open PR", "The pull request could not be opened."); + } + }; + + const content = ( + + {query.error !== null && graphs === null ? ( + {query.error} + ) : graphs === null ? ( + Loading task graphs… + ) : graphs.length === 0 ? ( + + No task graphs on this thread. + + ) : ( + graphs.map((graph) => { + const actions = taskGraphActions(graph); + const busy = pending.has(graph.id); + return ( + + + + + + {graph.title} + + + + {taskGraphSummaryLabel(graph)} · from {graph.baseRef} + + + {(actions.run && canRun) || (actions.cancel && canCancel) ? ( + + {actions.run && canRun ? ( + onRun(graph)} /> + ) : null} + {actions.cancel && canCancel ? ( + onCancel(graph)} + /> + ) : null} + + ) : null} + {taskGraphLayers(graph.nodes).map((layer, index) => ( + + + Step {index + 1} + + {layer.map((node) => { + const nodeActions = taskGraphNodeActions(graph, node); + return ( + openNodeThread(node)} + onOpenPullRequest={(url) => void openPullRequest(url)} + onCancelBranch={() => onCancelBranch(graph, node)} + onRetry={() => onRetry(graph, node)} + /> + ); + })} + + ))} + + ); + }) + )} + + ); + + if (Platform.OS === "ios") { + // A plain formSheet screen never renders a stack header, so it comes from + // a nested native stack inside the sheet (same shape as the agents sheet). + return ( + + + + {content} + + + + + ); + } + + return ( + + navigation.goBack()} /> + {content} + + ); +} + +function NodeRow(props: { + readonly node: TaskGraphNode; + /** Model and machine, empty when neither is known. */ + readonly placement: string; + readonly busy: boolean; + readonly canCancelBranch: boolean; + readonly canRetry: boolean; + readonly onOpen: () => void; + readonly onOpenPullRequest: (url: string) => void; + readonly onCancelBranch: () => void; + readonly onRetry: () => void; +}) { + const { node } = props; + const tone = taskGraphNodeTone(node.status); + const pullRequestUrl = node.pullRequestResult?.url ?? null; + const pullRequestError = + node.pullRequestResult?.status === "failed" ? node.pullRequestResult.error : null; + const detail = node.status === "succeeded" ? pullRequestError : (node.error ?? pullRequestError); + // Read once: "today" versus "tomorrow" only needs to be right when the sheet opens. + const [now] = useState(Date.now); + const wait = taskGraphNodeWaitDetail(node, now); + const hasActions = pullRequestUrl !== null || props.canCancelBranch || props.canRetry; + + const summary = ( + + + + + {node.title} + + + {taskGraphNodeStatusLabel(node.status)} + + + {wait !== null ? ( + + {wait} + + ) : null} + {props.placement !== "" ? ( + + {props.placement} + + ) : null} + {node.branch !== null ? ( + + {node.branch} + + ) : null} + {detail ? ( + + {detail} + + ) : null} + + ); + + return ( + + {node.threadId === null ? ( + {summary} + ) : ( + + {summary} + + )} + {hasActions ? ( + + {pullRequestUrl !== null ? ( + props.onOpenPullRequest(pullRequestUrl)} /> + ) : null} + {props.canRetry ? ( + + ) : null} + {props.canCancelBranch ? ( + + ) : null} + + ) : null} + + ); +} + +function NodeAction(props: { + readonly label: string; + readonly disabled?: boolean; + readonly onPress: () => void; +}) { + return ( + + {props.label} + + ); +} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index e4f489115f3a..84053d13dbc8 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -144,6 +144,8 @@ import { import { ComposerPopoverHost } from "./ComposerPopoverHost"; import { ThreadFeed, type ThreadFeedHistoryControls } from "./ThreadFeed"; import { useThreadTurnSubagents } from "./ThreadAgentsSheet"; +import { useThreadTaskGraphs } from "./TaskGraphSheet"; +import { resolveTaskGraphSegment } from "./task-graph-presentation"; import { ComposerQueuedEditBanner } from "./ComposerQueuedEdit"; import { useThreadQueuedCount } from "./ThreadQueueControl"; import type { ThreadContentPresentation } from "./threadContentPresentation"; @@ -420,6 +422,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread threadId: props.selectedThread.id, }); const agentsSegment = resolveSubagentPillSegment(turnSubagents); + const taskGraphSegment = resolveTaskGraphSegment( + useThreadTaskGraphs({ environmentId: props.environmentId, threadId: props.selectedThread.id }), + ); const composerEditorRef = useRef(null); // A provider-native subagent shows status instead of a composer. const isProviderSubagent = isProviderNativeSubagentThread(props.selectedThread.source); @@ -568,6 +573,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread showWorkingControl || queuedCount > 0 || agentsSegment !== null || + taskGraphSegment !== null || devicePreviews.length > 0 || browserTabs.tabs.length > 0 || props.connectionStateLabel !== "connected" || @@ -1266,6 +1272,14 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread threadId: props.selectedThread.id, }); }} + taskGraph={taskGraphSegment} + onOpenTaskGraph={() => { + Keyboard.dismiss(); + navigation.navigate("ThreadTaskGraphs", { + environmentId: props.environmentId, + threadId: props.selectedThread.id, + }); + }} queuedCount={queuedCount} onOpenQueue={() => { Keyboard.dismiss(); diff --git a/apps/mobile/src/features/threads/floating-working-control.tsx b/apps/mobile/src/features/threads/floating-working-control.tsx index 8672e3834521..9218f4fb3e84 100644 --- a/apps/mobile/src/features/threads/floating-working-control.tsx +++ b/apps/mobile/src/features/threads/floating-working-control.tsx @@ -29,6 +29,8 @@ import { NATIVE_LIQUID_GLASS_SUPPORTED } from "../../native/native-glass"; import { BrowserPreviewButton } from "../browser/browser-preview-button"; import { DevicePreviewButton } from "../devices/device-preview-button"; import type { FloatingWorkingStatus } from "./floating-working-status"; +import { SubagentStatusDot } from "./SubagentStatusDot"; +import type { TaskGraphSegment } from "./task-graph-presentation"; import { ShimmeringWorkContent } from "./thread-work-log"; import { useVisibleSecondClock } from "./use-visible-second-clock"; @@ -75,6 +77,8 @@ export function FloatingWorkingControl(props: { readonly onScrollToEnd: () => void; readonly agents: SubagentPillSegment | null; readonly onOpenAgents: () => void; + readonly taskGraph: TaskGraphSegment | null; + readonly onOpenTaskGraph: () => void; readonly queuedCount: number; readonly onOpenQueue: () => void; /** Extra distance to rise above the anchor, e.g. an overlay card's coverage. */ @@ -84,11 +88,14 @@ export function FloatingWorkingControl(props: { const [overlayWidth, setOverlayWidth] = useState(windowWidth); const [queueWidth, setQueueWidth] = useState(0); const [agentsWidth, setAgentsWidth] = useState(0); + const [taskGraphWidth, setTaskGraphWidth] = useState(0); const hasQueue = props.queuedCount > 0; const hasPreview = props.devicePreview !== null || props.browserPreview !== null; const [previewWidth, setPreviewWidth] = useState(0); const agents = props.agents; const hasAgents = agents !== null; + const taskGraph = props.taskGraph; + const hasTaskGraph = taskGraph !== null; // Segments keep their measured width; only the status label absorbs the // remainder, so a long "Working 12m 04s" truncates before a count does. const labelWidth = Math.max( @@ -98,6 +105,7 @@ export function FloatingWorkingControl(props: { 32 - (hasQueue ? queueWidth : 0) - (hasAgents ? agentsWidth : 0) - + (hasTaskGraph ? taskGraphWidth : 0) - (hasPreview ? previewWidth : 0), ); const separationProgress = useSharedValue(props.showScrollToEnd ? 1 : 0); @@ -135,7 +143,7 @@ export function FloatingWorkingControl(props: { // Forget the width while no label is shown so the next one appears at its // own size instead of animating from the previous label's. const hasStatus = props.status !== null; - const hasCapsule = hasStatus || hasQueue || hasAgents || hasPreview; + const hasCapsule = hasStatus || hasQueue || hasAgents || hasTaskGraph || hasPreview; useEffect(() => { if (!hasStatus) { measuredWidthRef.current = null; @@ -159,12 +167,14 @@ export function FloatingWorkingControl(props: { const previewCompact = hasStatus || hasAgents || + hasTaskGraph || hasQueue || (props.devicePreview !== null && props.browserPreview !== null); - // The queue, agents, questions, and reconnect labels have separate tap targets. + // The queue, agents, task graph, questions, and reconnect labels have separate tap targets. const statusInteractive = props.status?.kind === "connection" || props.status?.kind === "child-input"; - const capsuleInteractive = statusInteractive || hasQueue || hasAgents || hasPreview; + const capsuleInteractive = + statusInteractive || hasQueue || hasAgents || hasTaskGraph || hasPreview; // The host stays centered on the capsule, but its measurement constraint // comes from the overlay, independent of the capsule's current width. const statusContent = @@ -225,6 +235,26 @@ export function FloatingWorkingControl(props: { ) : null} + {taskGraph !== null ? ( + setTaskGraphWidth(event.nativeEvent.layout.width)} + // Capped like the queue so agents, graph and queue still fit on a narrow screen. + style={{ maxWidth: Math.min(overlayWidth, windowWidth) * 0.4 }} + className="h-11 flex-row items-center gap-1.5 px-3 active:opacity-70" + > + {hasStatus || hasPreview || hasAgents ? ( + + ) : null} + + + {taskGraph.label} + + + ) : null} {hasQueue ? ( - {hasStatus || hasPreview || hasAgents ? ( + {hasStatus || hasPreview || hasAgents || hasTaskGraph ? ( ) : null} diff --git a/apps/mobile/src/features/threads/task-graph-presentation.test.ts b/apps/mobile/src/features/threads/task-graph-presentation.test.ts new file mode 100644 index 000000000000..2eeaa646b964 --- /dev/null +++ b/apps/mobile/src/features/threads/task-graph-presentation.test.ts @@ -0,0 +1,182 @@ +import type { TaskGraph, TaskGraphNode } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + resolveTaskGraphSegment, + sortTaskGraphs, + taskGraphActions, + taskGraphNodeActions, + taskGraphNodeWaitDetail, + taskGraphSummaryLabel, +} from "./task-graph-presentation"; + +const node = (key: string, status: TaskGraphNode["status"], dependsOn: string[] = []) => + ({ key, status, dependsOn, title: key }) as unknown as TaskGraphNode; + +const graph = ( + status: TaskGraph["status"], + nodes: TaskGraphNode[], + overrides: Partial = {}, +) => + ({ + id: `graph-${status}`, + title: "Ship auth", + status, + nodes, + updatedAt: "2026-10-01T00:00:00.000Z", + ...overrides, + }) as unknown as TaskGraph; + +describe("taskGraphSummaryLabel", () => { + it("counts finished tasks while running", () => { + expect( + taskGraphSummaryLabel( + graph("running", [node("a", "succeeded"), node("b", "running"), node("c", "pending")]), + ), + ).toBe("Running · 1 of 3 done"); + }); + + it("counts waiting tasks while running", () => { + expect( + taskGraphSummaryLabel(graph("running", [node("a", "succeeded"), node("b", "waiting")])), + ).toBe("Running · 1 of 2 done · 1 waiting"); + }); + + it("names failures on a failed graph", () => { + expect( + taskGraphSummaryLabel(graph("failed", [node("a", "failed"), node("b", "skipped", ["a"])])), + ).toBe("Failed · 1 failed"); + }); + + it("counts tasks on a draft", () => { + expect(taskGraphSummaryLabel(graph("draft", [node("a", "pending")]))).toBe("Draft · 1 task"); + }); +}); + +describe("resolveTaskGraphSegment", () => { + it("is absent without graphs", () => { + expect(resolveTaskGraphSegment(null)).toBeNull(); + expect(resolveTaskGraphSegment([])).toBeNull(); + }); + + it("prefers an open graph over a newer finished one", () => { + const segment = resolveTaskGraphSegment([ + graph("succeeded", [node("a", "succeeded")], { updatedAt: "2026-10-09T00:00:00.000Z" }), + graph("running", [node("a", "succeeded"), node("b", "running")]), + ]); + expect(segment).toMatchObject({ label: "Graph 1/2", tone: "working" }); + }); + + it("stays reachable once every graph has finished", () => { + expect(resolveTaskGraphSegment([graph("failed", [node("a", "failed")])])).toMatchObject({ + label: "Graph failed", + tone: "failed", + }); + }); + + it("counts several open graphs", () => { + expect( + resolveTaskGraphSegment([ + graph("draft", [node("a", "pending")]), + graph("running", [node("a", "running")]), + ])?.label, + ).toBe("2 graphs"); + }); +}); + +describe("sortTaskGraphs", () => { + it("puts running before draft before finished, newest first within each", () => { + const sorted = sortTaskGraphs([ + graph("succeeded", [], { id: "old" as never, updatedAt: "2026-10-01T00:00:00.000Z" }), + graph("draft", [], { id: "draft" as never }), + graph("failed", [], { id: "new" as never, updatedAt: "2026-10-05T00:00:00.000Z" }), + graph("running", [], { id: "running" as never }), + ]); + expect(sorted.map((entry) => entry.id)).toEqual(["running", "draft", "new", "old"]); + }); +}); + +describe("taskGraphActions", () => { + it("runs only drafts and cancels only open graphs", () => { + expect(taskGraphActions({ status: "draft" })).toEqual({ run: true, cancel: true }); + expect(taskGraphActions({ status: "running" })).toEqual({ run: false, cancel: true }); + expect(taskGraphActions({ status: "failed" })).toEqual({ run: false, cancel: false }); + }); +}); + +describe("taskGraphNodeActions", () => { + it("cancels the branch of unfinished nodes in an open graph", () => { + const nodes = [node("a", "running"), node("b", "pending", ["a"])]; + expect(taskGraphNodeActions(graph("running", nodes), nodes[0]!)).toEqual({ + cancelBranch: true, + retry: false, + }); + expect(taskGraphNodeActions(graph("running", nodes), nodes[1]!).cancelBranch).toBe(true); + }); + + it("cancels the branch of a waiting node, started or not", () => { + const nodes = [node("a", "waiting")]; + expect(taskGraphNodeActions(graph("running", nodes), nodes[0]!)).toEqual({ + cancelBranch: true, + retry: false, + }); + }); + + it("retries failed nodes, including in a finished graph", () => { + const nodes = [node("a", "failed")]; + expect(taskGraphNodeActions(graph("failed", nodes), nodes[0]!)).toEqual({ + cancelBranch: false, + retry: true, + }); + }); + + it("offers no retry for a skipped node whose dependency still blocks it", () => { + const nodes = [node("a", "failed"), node("b", "skipped", ["a"])]; + expect(taskGraphNodeActions(graph("failed", nodes), nodes[1]!).retry).toBe(false); + }); + + it("retries a skipped node once its dependency has succeeded", () => { + const nodes = [node("a", "succeeded"), node("b", "skipped", ["a"])]; + expect(taskGraphNodeActions(graph("failed", nodes), nodes[1]!).retry).toBe(true); + }); + + it("takes no edits on a cancelled graph", () => { + const nodes = [node("a", "cancelled")]; + expect(taskGraphNodeActions(graph("cancelled", nodes), nodes[0]!)).toEqual({ + cancelBranch: false, + retry: false, + }); + }); +}); + +describe("taskGraphNodeWaitDetail", () => { + const now = new Date(2026, 9, 10, 9, 0).getTime(); + const waitUntil = new Date(2026, 9, 10, 15, 40).toISOString(); + const time = new Date(waitUntil).toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }); + const waiting = (overrides: Partial) => + ({ status: "waiting", waitUntil, threadId: null, ...overrides }) as TaskGraphNode; + + it("says when a scheduled node starts", () => { + expect(taskGraphNodeWaitDetail(waiting({ waitReason: "scheduled" }), now)).toBe( + `Starts ${time}`, + ); + }); + + it("says whether a limited node starts or continues its thread after the reset", () => { + expect(taskGraphNodeWaitDetail(waiting({ waitReason: "usage_limit" }), now)).toBe( + `Usage limit reached — starts after the reset, ${time}`, + ); + expect( + taskGraphNodeWaitDetail( + waiting({ waitReason: "usage_limit", threadId: "thread-a" as TaskGraphNode["threadId"] }), + now, + ), + ).toBe(`Usage limit reached — continues on its thread after the reset, ${time}`); + }); + + it("names the day once the wait ends after today", () => { + const tomorrow = waiting({ waitReason: "scheduled" }); + expect(taskGraphNodeWaitDetail(tomorrow, now - 86_400_000)).toBe(`Starts tomorrow at ${time}`); + expect(taskGraphNodeWaitDetail({ ...tomorrow, status: "pending" }, now)).toBeNull(); + }); +}); diff --git a/apps/mobile/src/features/threads/task-graph-presentation.ts b/apps/mobile/src/features/threads/task-graph-presentation.ts new file mode 100644 index 000000000000..27af27d669f1 --- /dev/null +++ b/apps/mobile/src/features/threads/task-graph-presentation.ts @@ -0,0 +1,204 @@ +import type { + TaskGraph, + TaskGraphNode, + TaskGraphNodeStatus, + TaskGraphStatus, +} from "@t3tools/contracts"; +import { isActiveTaskGraphNodeStatus } from "@t3tools/shared/taskGraph"; + +import type { SubagentRowTone } from "./threadAgentsPresentation"; + +const BLOCKING_STATUSES: ReadonlySet = new Set([ + "failed", + "cancelled", + "skipped", +]); + +export const isOpenTaskGraph = (graph: Pick): boolean => + graph.status === "draft" || graph.status === "running"; + +export function taskGraphNodeTone(status: TaskGraphNodeStatus): SubagentRowTone { + if (isActiveTaskGraphNodeStatus(status)) return "working"; + if (status === "succeeded") return "completed"; + if (status === "failed") return "failed"; + return "stopped"; +} + +export function taskGraphTone(status: TaskGraphStatus): SubagentRowTone { + if (status === "running") return "working"; + if (status === "succeeded") return "completed"; + if (status === "failed") return "failed"; + return "stopped"; +} + +export function taskGraphNodeStatusLabel(status: TaskGraphNodeStatus): string { + switch (status) { + case "pending": + return "Pending"; + case "waiting": + return "Waiting"; + case "running": + return "Running"; + case "delivering": + return "Opening PR"; + case "succeeded": + return "Done"; + case "failed": + return "Failed"; + case "cancelled": + return "Cancelled"; + case "skipped": + return "Skipped"; + } +} + +const localDayStart = (date: Date) => + new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime(); + +/** An upcoming instant: "3:40 PM" today, "tomorrow at 3:40 PM", else "Oct 13, 3:40 PM". */ +function formatTaskGraphWaitTime(iso: string, now: number): string { + const at = new Date(iso); + if (!Number.isFinite(at.getTime())) return ""; + const time = at.toLocaleTimeString([], { hour: "numeric", minute: "2-digit" }); + // Round so DST-shifted 23/25 hour days still count as whole days. + const days = Math.round((localDayStart(at) - localDayStart(new Date(now))) / 86_400_000); + if (days <= 0) return time; + if (days === 1) return `tomorrow at ${time}`; + const date = at.toLocaleDateString([], { + month: "short", + day: "numeric", + ...(at.getFullYear() !== new Date(now).getFullYear() ? { year: "numeric" as const } : {}), + }); + return `${date}, ${time}`; +} + +/** Why a waiting node waits and what happens next, as on web. Null when it is not waiting. */ +export function taskGraphNodeWaitDetail( + node: Pick, + now: number, +): string | null { + if (node.status !== "waiting") return null; + const at = node.waitUntil === null ? null : formatTaskGraphWaitTime(node.waitUntil, now); + if (node.waitReason === "usage_limit") { + const next = + node.threadId === null ? "starts after the reset" : "continues on its thread after the reset"; + return `Usage limit reached — ${next}${at === null ? "" : `, ${at}`}`; + } + return at === null ? "Waiting to start" : `Starts ${at}`; +} + +export function taskGraphStatusLabel(status: TaskGraphStatus): string { + switch (status) { + case "draft": + return "Draft"; + case "running": + return "Running"; + case "succeeded": + return "Done"; + case "failed": + return "Failed"; + case "cancelled": + return "Cancelled"; + } +} + +/** One-line status for a graph, such as "Running · 2 of 5 done". */ +export function taskGraphSummaryLabel(graph: Pick): string { + const total = graph.nodes.length; + const tasks = `${total} ${total === 1 ? "task" : "tasks"}`; + if (graph.status === "draft" || graph.status === "succeeded") { + return `${taskGraphStatusLabel(graph.status)} · ${tasks}`; + } + if (graph.status === "running") { + const done = graph.nodes.filter((node) => node.status === "succeeded").length; + const waiting = graph.nodes.filter((node) => node.status === "waiting").length; + return `Running · ${done} of ${total} done${waiting > 0 ? ` · ${waiting} waiting` : ""}`; + } + const failed = graph.nodes.filter((node) => node.status === "failed").length; + return failed > 0 + ? `${taskGraphStatusLabel(graph.status)} · ${failed} failed` + : `${taskGraphStatusLabel(graph.status)} · ${tasks}`; +} + +/** Open graphs first (running before draft), then the most recently changed. */ +export function sortTaskGraphs(graphs: ReadonlyArray): ReadonlyArray { + const rank = (graph: TaskGraph) => + graph.status === "running" ? 0 : graph.status === "draft" ? 1 : 2; + return [...graphs].sort( + (left, right) => rank(left) - rank(right) || right.updatedAt.localeCompare(left.updatedAt), + ); +} + +export interface TaskGraphSegment { + readonly label: string; + readonly accessibilityLabel: string; + readonly tone: SubagentRowTone; +} + +/** + * The thread's task graph entry point. Finished graphs keep it visible so a + * failed node can still be retried and its pull requests reached. + */ +export function resolveTaskGraphSegment( + graphs: ReadonlyArray | null, +): TaskGraphSegment | null { + if (graphs === null || graphs.length === 0) return null; + const open = graphs.filter(isOpenTaskGraph); + if (open.length > 1) { + return { + label: `${open.length} graphs`, + accessibilityLabel: `${open.length} open task graphs`, + tone: open.some((graph) => graph.status === "running") ? "working" : "stopped", + }; + } + const graph = sortTaskGraphs(graphs)[0]!; + const label = + graph.status === "running" + ? `Graph ${graph.nodes.filter((node) => node.status === "succeeded").length}/${graph.nodes.length}` + : `Graph ${taskGraphStatusLabel(graph.status).toLowerCase()}`; + return { + label, + accessibilityLabel: `Task graph ${graph.title}, ${taskGraphSummaryLabel(graph)}`, + tone: taskGraphTone(graph.status), + }; +} + +export interface TaskGraphActions { + readonly run: boolean; + readonly cancel: boolean; +} + +export const taskGraphActions = (graph: Pick): TaskGraphActions => ({ + run: graph.status === "draft", + cancel: isOpenTaskGraph(graph), +}); + +export interface TaskGraphNodeActions { + readonly cancelBranch: boolean; + readonly retry: boolean; +} + +/** + * What a node offers. A cancelled graph takes no edits. Retrying a skipped + * node whose dependency still has not succeeded would only skip it again, so + * that retry belongs on the dependency instead. + */ +export function taskGraphNodeActions( + graph: Pick, + node: Pick, +): TaskGraphNodeActions { + if (graph.status === "cancelled") return { cancelBranch: false, retry: false }; + const cancelBranch = + isOpenTaskGraph(graph) && + (node.status === "pending" || + node.status === "waiting" || + isActiveTaskGraphNodeStatus(node.status)); + if (node.status === "failed" || node.status === "cancelled") return { cancelBranch, retry: true }; + if (node.status !== "skipped") return { cancelBranch, retry: false }; + const statusByKey = new Map(graph.nodes.map((other) => [other.key, other.status])); + const blocked = node.dependsOn.some((key) => { + const status = statusByKey.get(key); + return status !== undefined && BLOCKING_STATUSES.has(status); + }); + return { cancelBranch, retry: !blocked }; +} diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 5b8d483dc102..9d7cb6d32e12 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -13,6 +13,7 @@ import { AuthSettingsWriteScope, AuthProvidersManageScope, AuthEnvironmentMaintainScope, + TASK_GRAPH_PEER_SCOPES, AuthFilesystemReadScope, AuthFilesystemWriteScope, AuthDiagnosticsReadScope, @@ -115,6 +116,8 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverGetBackgroundPolicy]: AuthOrchestrationReadScope, [WS_METHODS.scheduledTasksList]: AuthOrchestrationReadScope, [WS_METHODS.scheduledTasksSubscribe]: AuthOrchestrationReadScope, + [WS_METHODS.taskGraphsSubscribe]: AuthOrchestrationReadScope, + [WS_METHODS.taskGraphPeersSubscribe]: AuthOrchestrationReadScope, [WS_METHODS.secretsAnswerRequest]: AuthOrchestrationOperateScope, // Delivery logs hold request bodies, so they need the same scope as the URL. [WS_METHODS.scheduledTasksListWebhookDeliveries]: AuthOrchestrationOperateScope, @@ -266,6 +269,10 @@ const requiredScopesForRpcCall = ( : AuthOrchestrationReadScope, ]; } + if (method === WS_METHODS.taskGraphPeersIssueGrant) { + // A grant carries these scopes, so never hand out more than the caller holds itself. + return [AuthEnvironmentMaintainScope, ...TASK_GRAPH_PEER_SCOPES]; + } if (method === WS_METHODS.serverUpdateSettings) return requiredScopesForSettingsUpdate(payload); if (method === WS_METHODS.projectsMutate) { return requiredScopesForProjectMutation(decodeProjectMutation(payload)); diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index 6079ee714f69..bc5a3f868808 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -2216,6 +2216,7 @@ export const make = Effect.gen(function* () { cwd: string, fallbackBranch: string | null, emit: GitActionProgressEmitter, + baseBranchOverride?: string, ) { const provider = yield* sourceControlProvider(cwd); const terms = getChangeRequestTerminologyForKind(provider.kind); @@ -2253,7 +2254,9 @@ export const make = Effect.gen(function* () { }; } - const baseBranch = yield* resolveBaseBranch(cwd, branch, details.upstreamRef, headContext); + const baseBranch = + baseBranchOverride ?? + (yield* resolveBaseBranch(cwd, branch, details.upstreamRef, headContext)); yield* emit({ kind: "phase_started", phase: "pr", @@ -3010,7 +3013,13 @@ export const make = Effect.gen(function* () { .pipe( Effect.tap(() => Ref.set(currentPhase, Option.some("pr"))), Effect.flatMap(() => - runPrStep(textGenerationSettings, input.cwd, currentBranch, progress.emit), + runPrStep( + textGenerationSettings, + input.cwd, + currentBranch, + progress.emit, + input.baseBranch, + ), ), ) : { status: "skipped_not_requested" as const }; diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 0e2b52696712..26d7181d5d6a 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -47,6 +47,8 @@ import { } from "./toolkits/preview/tools.ts"; import * as WorktreeHandlers from "./toolkits/worktree/handlers.ts"; import { WorktreeToolkit } from "./toolkits/worktree/tools.ts"; +import * as TaskGraphHandlers from "./toolkits/taskGraph/handlers.ts"; +import { TaskGraphToolkit } from "./toolkits/taskGraph/tools.ts"; import * as WorktreeMcpService from "./WorktreeMcpService.ts"; import * as PullRequestsHandlers from "./toolkits/pullRequests/handlers.ts"; import { PullRequestsToolkit } from "./toolkits/pullRequests/tools.ts"; @@ -816,6 +818,8 @@ export const layerOrchestratorToolkit = toolkitRegistration( export const layerThreadToolkit = toolkitRegistration(ThreadToolkit, ThreadHandlers.layer); +export const layerTaskGraphToolkit = toolkitRegistration(TaskGraphToolkit, TaskGraphHandlers.layer); + const layerWorktreeToolkitRegistration = toolkitRegistration( WorktreeToolkit, WorktreeHandlers.layer, @@ -870,6 +874,7 @@ export const layer = Layer.mergeAll( layerPreviewToolkit, layerOrchestratorToolkit, layerThreadToolkit, + layerTaskGraphToolkit, layerAttachmentToolkit, layerProjectRegistration, layerEnvironmentToolkit, diff --git a/apps/server/src/mcp/toolkits/taskGraph/handlers.ts b/apps/server/src/mcp/toolkits/taskGraph/handlers.ts new file mode 100644 index 000000000000..c3ed8ebca62c --- /dev/null +++ b/apps/server/src/mcp/toolkits/taskGraph/handlers.ts @@ -0,0 +1,103 @@ +import { OrchestratorMcpFailure, type TaskGraphId, type ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as McpToolAccess from "../../McpToolAccess.ts"; +import * as ServerEnvironment from "../../../environment/ServerEnvironment.ts"; +import * as TaskGraphPeers from "../../../taskGraph/TaskGraphPeers.ts"; +import * as TaskGraphService from "../../../taskGraph/TaskGraphService.ts"; +import { TaskGraphToolkit } from "./tools.ts"; + +const failure = (code: "invalid_request" | "orchestration_error", message: string) => + new OrchestratorMcpFailure({ code, message }); + +/** The calling thread; the access gate already refused callers without one. */ +const callerThread = Effect.gen(function* () { + const scope = yield* McpInvocationContext.McpInvocationContext; + if (scope.thread === undefined) { + return yield* failure( + "invalid_request", + "Task graphs need an agent running inside a T3 thread.", + ); + } + return scope.thread.threadId; +}); + +const orchestrationError = (error: { readonly message: string }) => + failure("orchestration_error", error.message); + +/** A graph the calling thread created; agents only steer their own graphs. */ +const ownGraph = (graphId: TaskGraphId, threadId: ThreadId) => + Effect.gen(function* () { + const service = yield* TaskGraphService.TaskGraphService; + const graph = yield* service.get(graphId).pipe(Effect.mapError(orchestrationError)); + if (graph.threadId !== threadId) { + return yield* failure("invalid_request", "That task graph belongs to another thread."); + } + return service; + }); + +const handlers = { + task_graph_create: McpToolAccess.actsAsCaller((input) => + Effect.gen(function* () { + const threadId = yield* callerThread; + const service = yield* TaskGraphService.TaskGraphService; + const graph = yield* service + .create({ ...input, threadId }) + .pipe(Effect.mapError(orchestrationError)); + return { graph }; + }), + ), + task_graph_list: McpToolAccess.readsAsCaller(() => + Effect.gen(function* () { + const threadId = yield* callerThread; + const service = yield* TaskGraphService.TaskGraphService; + const graphs = yield* service + .listForThread(threadId) + .pipe(Effect.mapError(orchestrationError)); + const local = yield* (yield* ServerEnvironment.ServerEnvironment).getDescriptor; + const { peers } = yield* (yield* TaskGraphPeers.TaskGraphPeers).list.pipe( + Effect.mapError(orchestrationError), + ); + return { + graphs, + machines: [ + { + environmentId: local.environmentId, + label: local.label, + thisMachine: true, + connected: true, + }, + ...peers.map((peer) => ({ + environmentId: peer.environmentId, + label: peer.label, + thisMachine: false, + connected: peer.status === "connected", + })), + ], + }; + }), + ), + task_graph_edit: McpToolAccess.actsAsCaller(({ graphId, edits }) => + Effect.gen(function* () { + const service = yield* ownGraph(graphId, yield* callerThread); + return { + graph: yield* service.edit(graphId, edits).pipe(Effect.mapError(orchestrationError)), + }; + }), + ), + task_graph_run: McpToolAccess.actsAsCaller(({ graphId }) => + Effect.gen(function* () { + const service = yield* ownGraph(graphId, yield* callerThread); + return { graph: yield* service.run(graphId).pipe(Effect.mapError(orchestrationError)) }; + }), + ), + task_graph_cancel: McpToolAccess.actsAsCaller(({ graphId }) => + Effect.gen(function* () { + const service = yield* ownGraph(graphId, yield* callerThread); + return { graph: yield* service.cancel(graphId).pipe(Effect.mapError(orchestrationError)) }; + }), + ), +} satisfies McpToolAccess.Handlers; + +export const layer = McpToolAccess.toLayer(TaskGraphToolkit, handlers); diff --git a/apps/server/src/mcp/toolkits/taskGraph/tools.ts b/apps/server/src/mcp/toolkits/taskGraph/tools.ts new file mode 100644 index 000000000000..cbc8c0ec3dc4 --- /dev/null +++ b/apps/server/src/mcp/toolkits/taskGraph/tools.ts @@ -0,0 +1,130 @@ +import { + EnvironmentId, + MAX_TASK_GRAPH_NODES, + OrchestratorMcpFailure, + TaskGraph, + TaskGraphEdit, + TaskGraphId, + TaskGraphNodeInput, + TrimmedNonEmptyString, +} from "@t3tools/contracts"; +import { Tool, Toolkit } from "effect/ai"; +import * as Schema from "effect/Schema"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts"; +import * as ServerEnvironment from "../../../environment/ServerEnvironment.ts"; +import * as TaskGraphPeers from "../../../taskGraph/TaskGraphPeers.ts"; +import * as TaskGraphService from "../../../taskGraph/TaskGraphService.ts"; + +const dependencies = [ + McpInvocationContext.McpInvocationContext, + ThreadManagementService.ThreadManagementService, + TaskGraphService.TaskGraphService, + TaskGraphPeers.TaskGraphPeers, + ServerEnvironment.ServerEnvironment, +]; + +const GraphResult = Schema.Struct({ graph: TaskGraph }); +const GraphTarget = Schema.Struct({ graphId: TaskGraphId }); + +export const TaskGraphCreateTool = Tool.make("task_graph_create", { + description: + "Needs an agent running inside a T3 thread. Plan work as a graph of agent tasks and run it. Each node runs as its own T3 thread, in parallel where dependencies allow, and receives every dependency's final summary. Write each prompt so an agent with no access to this conversation can do it. " + + "WORKSPACES: by default a node gets its own git worktree, branching from its first dependency's branch (or baseRef, default this thread's branch). workspace 'dependency' continues in the first dependency's worktree and branch instead, so several nodes build one branch in turn. workspace 'root' works in the project folder with no branch, commit or PR; use it for read-only work such as reviews and audits. " + + "PULL REQUESTS: a succeeding node's work is committed. Nodes nothing depends on also push and open a PR; set pullRequest per node to change that. A PR targets the nearest earlier node (following first dependencies) that has its own PR, else the graph base, so PRs only on branch ends give one PR each against the base, and PRs on inner nodes too give a stack. " + + "STACKED PRS: when the user asks for stacked PRs, do not do the work yourself one layer after another. Make each layer a node (or a short 'dependency' chain) with pullRequest true, each depending on the layer below, and put independent work for a layer in its own subtree that merges back before the next layer. One graph then produces the whole stack. " + + "MERGING: a node with several dependencies starts from the first one's branch and merges the others, resolving conflicts. Only add such a combining node when the user asks for the pieces to be reconciled or they cannot work apart; otherwise leave the branches independent, each ending in its own PR. " + + "MACHINES: set environmentId on a node to pin it to a machine listed by task_graph_list (only when the user asks); otherwise omit it and nodes are balanced across machines by load. A 'dependency' node always runs where its dependency ran. Set modelSelection on a node only when the user asks for a different model. " + + "TIMING: set startAt on a node to hold it until a time the user asks for, such as overnight. A node never starts into a used-up usage limit: it waits for the reset. A node whose run is stopped by a usage limit waits too, and continues on its own thread at the reset, so do not retry or replace it. " + + "RUNNING: if the user asked to see, review or edit the plan first, pass run=false so it waits as a draft they can edit in the chat; if they asked to just do it, pass run=true; otherwise omit run to follow their setting. Change a graph later with task_graph_edit.", + parameters: Schema.Struct({ + title: TrimmedNonEmptyString.check(Schema.isMaxLength(120)), + nodes: Schema.Array(TaskGraphNodeInput).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_TASK_GRAPH_NODES), + ), + baseRef: Schema.optional(TrimmedNonEmptyString), + run: Schema.optional(Schema.Boolean), + }), + success: GraphResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies, +}) + .annotate(Tool.Title, "Create a task graph") + // Running nodes can commit, push and open pull requests. + .annotate(Tool.Destructive, true) + .annotate(Tool.OpenWorld, true); + +const TaskGraphListTool = Tool.make("task_graph_list", { + description: + "Needs an agent running inside a T3 thread. List the task graphs this thread created, with every node's status, thread, machine, branch, summary and pull request, and the machines nodes can run on (pass one's environmentId to pin a node there).", + success: Schema.Struct({ + graphs: Schema.Array(TaskGraph), + machines: Schema.Array( + Schema.Struct({ + environmentId: EnvironmentId, + label: Schema.String, + thisMachine: Schema.Boolean, + connected: Schema.Boolean, + }), + ), + }), + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies, +}) + .annotate(Tool.Title, "List task graphs") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true); + +const TaskGraphEditTool = Tool.make("task_graph_edit", { + description: + "Change a task graph from this thread, running or not. Edits apply in order and all or nothing. add_node adds work (it starts as soon as its dependencies succeeded); update_node and remove_node change nodes that have not started; cancel_branch cuts a node and everything below it, stopping any that are running; retry_node reruns a node that failed or was cancelled, along with what was skipped below it.", + parameters: Schema.Struct({ + graphId: TaskGraphId, + edits: Schema.Array(TaskGraphEdit).check(Schema.isMinLength(1)), + }), + success: GraphResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies, +}) + .annotate(Tool.Title, "Edit a task graph") + .annotate(Tool.Destructive, true); + +const TaskGraphRunTool = Tool.make("task_graph_run", { + description: + "Start a draft task graph from this thread. Only call this when the user asks to run it.", + parameters: GraphTarget, + success: GraphResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies, +}) + .annotate(Tool.Title, "Run a task graph") + // Running nodes can commit, push and open pull requests. + .annotate(Tool.Destructive, true) + .annotate(Tool.OpenWorld, true); + +const TaskGraphCancelTool = Tool.make("task_graph_cancel", { + description: + "Cancel a whole task graph from this thread, stopping every running node. Finished nodes and their pull requests stay.", + parameters: GraphTarget, + success: GraphResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies, +}) + .annotate(Tool.Title, "Cancel a task graph") + .annotate(Tool.Destructive, true); + +export const TaskGraphToolkit = Toolkit.make( + TaskGraphCreateTool, + TaskGraphListTool, + TaskGraphEditTool, + TaskGraphRunTool, + TaskGraphCancelTool, +); diff --git a/apps/server/src/mcp/toolkits/worktree/registration.test.ts b/apps/server/src/mcp/toolkits/worktree/registration.test.ts index c0006f7fe171..3f1ad32e7b4e 100644 --- a/apps/server/src/mcp/toolkits/worktree/registration.test.ts +++ b/apps/server/src/mcp/toolkits/worktree/registration.test.ts @@ -1,4 +1,6 @@ import * as Orchestrator from "../../../orchestration-v2/Orchestrator.ts"; +import * as TaskGraphPeers from "../../../taskGraph/TaskGraphPeers.ts"; +import * as TaskGraphService from "../../../taskGraph/TaskGraphService.ts"; import * as ProjectionStore from "../../../orchestration-v2/ProjectionStore.ts"; import * as DeviceService from "../../../device/DeviceService.ts"; import * as ServerConfig from "../../../config.ts"; @@ -37,6 +39,8 @@ import * as PreviewBrowser from "../../../preview/PreviewBrowser.ts"; const layerStubServices = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({}), + Layer.mock(TaskGraphService.TaskGraphService)({}), + Layer.mock(TaskGraphPeers.TaskGraphPeers)({}), Layer.mock(ProjectionStore.ProjectionStoreV2)({}), Layer.mock(DeviceService.DeviceService)({}), Layer.mock(ThreadManagementService.ThreadManagementService)({}), diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 167ce8d11fff..206dee76fbd2 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -84,6 +84,16 @@ const RPC_AGGREGATES = { [WS_METHODS.serverReportClientActivity]: "server", [WS_METHODS.serverReportHostPowerState]: "server", [WS_METHODS.serverGetBackgroundPolicy]: "server", + [WS_METHODS.taskGraphsSubscribe]: "taskGraphs", + [WS_METHODS.taskGraphsCreate]: "taskGraphs", + [WS_METHODS.taskGraphsEdit]: "taskGraphs", + [WS_METHODS.taskGraphsRun]: "taskGraphs", + [WS_METHODS.taskGraphsCancel]: "taskGraphs", + [WS_METHODS.taskGraphPeersSubscribe]: "taskGraphs", + [WS_METHODS.taskGraphPeersAdd]: "taskGraphs", + [WS_METHODS.taskGraphPeersRemove]: "taskGraphs", + [WS_METHODS.taskGraphPeersSetWeight]: "taskGraphs", + [WS_METHODS.taskGraphPeersIssueGrant]: "taskGraphs", [WS_METHODS.scheduledTasksList]: "scheduledTasks", [WS_METHODS.scheduledTasksSubscribe]: "scheduledTasks", [WS_METHODS.scheduledTasksUpsert]: "scheduledTasks", diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 623b1e084118..32b133c187c6 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -50,6 +50,8 @@ import * as ThreadLifecycleService from "./ThreadLifecycleService.ts"; import * as ThreadForkService from "./ThreadForkService.ts"; import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; +import * as TaskGraphPeers from "../taskGraph/TaskGraphPeers.ts"; +import * as TaskGraphService from "../taskGraph/TaskGraphService.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; /** The shared application event log and its command receipts. */ @@ -281,6 +283,18 @@ const layerScheduledTaskProvided = ScheduledTaskService.layer.pipe( ), ), ); +const layerTaskGraphPeersProvided = TaskGraphPeers.layer.pipe(Layer.provide(layerProjectService)); +const layerTaskGraphProvided = TaskGraphService.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerTaskGraphPeersProvided, + layerThreadLaunchProvided, + layerThreadManagementProvided, + layerOrchestratorProvided, + layerProjectService, + ), + ), +); const layerProviderContinuationWorkerProvided = ProviderContinuationService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -354,6 +368,8 @@ export const layerProduction = Layer.mergeAll( layerThreadLaunchProvided, layerThreadLifecycleProvided, layerScheduledTaskProvided, + layerTaskGraphProvided, + layerTaskGraphPeersProvided, layerSecretRequestsProvided, UsageLimitRecoveryWorker.layer.pipe( Layer.provide(Layer.mergeAll(ProjectionStore.layer, layerThreadManagementProvided)), diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 7990c53ac454..aba6b06205ac 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -75,6 +75,7 @@ import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts"; import Migration0059 from "./Migrations/059_McpAppModelContext.ts"; import Migration0060 from "./Migrations/060_ThreadSnapshotWindowIndexes.ts"; import Migration0061 from "./Migrations/061_PullRequestReviewGuides.ts"; +import Migration0062 from "./Migrations/062_TaskGraphs.ts"; /** * Migration loader with all migrations defined inline. @@ -150,6 +151,7 @@ export const migrationEntries = [ [59, "McpAppModelContext", Migration0059], [60, "ThreadSnapshotWindowIndexes", Migration0060], [61, "PullRequestReviewGuides", Migration0061], + [62, "TaskGraphs", Migration0062], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts index 7d5c845cbf22..537908dfeb55 100644 --- a/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts +++ b/apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts @@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => { Effect.sync(() => { assert.deepStrictEqual( migrationEntries.map(([id]) => id), - Array.from({ length: 61 }, (_, index) => index + 1), + Array.from({ length: 62 }, (_, index) => index + 1), ); }), ); @@ -33,6 +33,7 @@ layer("055_OrchestrationV2", (it) => { [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], [61, "PullRequestReviewGuides"], + [62, "TaskGraphs"], ]); assert.deepStrictEqual(yield* runMigrations(), []); @@ -60,6 +61,7 @@ layer("055_OrchestrationV2", (it) => { { migration_id: 59, name: "McpAppModelContext" }, { migration_id: 60, name: "ThreadSnapshotWindowIndexes" }, { migration_id: 61, name: "PullRequestReviewGuides" }, + { migration_id: 62, name: "TaskGraphs" }, ]); const tables = yield* sql<{ readonly name: string }>` diff --git a/apps/server/src/persistence/Migrations/062_TaskGraphs.ts b/apps/server/src/persistence/Migrations/062_TaskGraphs.ts new file mode 100644 index 000000000000..ca07eef0126c --- /dev/null +++ b/apps/server/src/persistence/Migrations/062_TaskGraphs.ts @@ -0,0 +1,41 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // A graph is written whole on every change; its nodes live in graph_json. + // Graphs are small (at most 48 nodes) and always read as a unit. + yield* sql` + CREATE TABLE IF NOT EXISTS task_graphs ( + graph_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + status TEXT NOT NULL, + graph_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `; + yield* sql` + CREATE INDEX IF NOT EXISTS idx_task_graphs_thread + ON task_graphs(thread_id, created_at) + `; + yield* sql` + CREATE INDEX IF NOT EXISTS idx_task_graphs_running + ON task_graphs(status) + WHERE status = 'running' + `; + + // Peer machines this server may place nodes on. The session token for each + // lives in the secret store, never in this table. + yield* sql` + CREATE TABLE IF NOT EXISTS task_graph_peers ( + environment_id TEXT PRIMARY KEY, + label TEXT NOT NULL, + http_base_url TEXT NOT NULL, + weight INTEGER NOT NULL, + added_at TEXT NOT NULL + ) + `; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 5881fdfda408..71c77fbc3dab 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -42,6 +42,7 @@ describe("V2 preview upgrade", () => { [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], [61, "PullRequestReviewGuides"], + [62, "TaskGraphs"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -126,6 +127,7 @@ describe("V2 preview upgrade", () => { [59, "McpAppModelContext"], [60, "ThreadSnapshotWindowIndexes"], [61, "PullRequestReviewGuides"], + [62, "TaskGraphs"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/taskGraph/TaskGraphPeers.ts b/apps/server/src/taskGraph/TaskGraphPeers.ts new file mode 100644 index 000000000000..e61fa61f660b --- /dev/null +++ b/apps/server/src/taskGraph/TaskGraphPeers.ts @@ -0,0 +1,829 @@ +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import { + CommandId, + EnvironmentId, + ORCHESTRATION_V2_WS_METHODS, + repositoryGroupingKeyOf, + TaskGraphError, + WS_METHODS, + WsRpcGroup, + type HostResourcesSnapshot, + type ModelSelection, + type OrchestrationProjectShell, + type OrchestrationV2ShellStreamItem, + type OrchestrationV2ThreadLaunchInput, + type OrchestrationV2ThreadShell, + type ProjectId, + type RuntimeMode, + type TaskGraph, + type TaskGraphNode, + type TaskGraphPeer, + type TaskGraphPeerGrant, + type TaskGraphPeerListResult, + TASK_GRAPH_PEER_SCOPES, + type ThreadId, +} from "@t3tools/contracts"; +import { resolveRemotePairingTarget } from "@t3tools/shared/remote"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FiberMap from "effect/FiberMap"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientRequest from "effect/http/HttpClientRequest"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as RpcClient from "effect/rpc/RpcClient"; +import * as RpcSerialization from "effect/rpc/RpcSerialization"; +import * as Socket from "effect/socket/Socket"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import type { NodeOutcome } from "./TaskGraphService.ts"; + +/** + * How a finished peer node's work leaves the peer: pushed so other machines can + * build on it, with a PR against `baseBranch` when the node opens one, or not + * at all for work in the project folder. + */ +export interface PeerDelivery { + readonly action: "commit_push" | "commit_push_pr" | "none"; + readonly baseBranch: string | null; +} + +export interface PeerNodeStart { + readonly environmentId: EnvironmentId; + readonly graph: TaskGraph; + readonly node: TaskGraphNode; + readonly threadId: ThreadId; + readonly workspaceStrategy: OrchestrationV2ThreadLaunchInput["workspaceStrategy"]; + readonly prompt: string; + readonly modelSelection: ModelSelection; + readonly runtimeMode: RuntimeMode; + readonly delivery: PeerDelivery; +} + +export interface PeerCandidate { + readonly environmentId: EnvironmentId; + readonly resources: HostResourcesSnapshot | null; + readonly receivedAt: number; + /** Zero never takes balanced work, but a node pinned there still runs there. */ + readonly weight: number; + /** Nodes this machine runs there now. */ + readonly running: number; +} + +/** + * Other machines this environment may run task graph nodes on. The graph + * service asks for placement candidates, hands over nodes placed on a peer, + * and hears back through `completions` when one ends. + */ +export class TaskGraphPeers extends Context.Service< + TaskGraphPeers, + { + /** Whether any peer is paired; branches are then pushed so other machines can reach them. */ + readonly hasPeers: Effect.Effect; + /** Connected peers hosting the same repository, with fresh load, under `maxNodesPerPeer`. */ + /** Connected peers with the project open, whatever their load; the caller applies limits. */ + readonly candidates: (input: { + readonly projectId: ProjectId; + }) => Effect.Effect>; + readonly startNode: (input: PeerNodeStart) => Effect.Effect; + /** Watches a node started before a restart. */ + readonly track: (input: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + readonly delivery: PeerDelivery; + }) => Effect.Effect; + readonly interruptNode: (input: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; + }) => Effect.Effect; + readonly completions: Stream.Stream<{ + readonly threadId: ThreadId; + readonly outcome: NodeOutcome; + }>; + readonly list: Effect.Effect; + readonly subscribe: Stream.Stream; + readonly add: (input: { + readonly pairingUrl: string; + readonly label?: string | undefined; + }) => Effect.Effect; + readonly remove: ( + environmentId: EnvironmentId, + ) => Effect.Effect; + /** A grant another machine redeems to run nodes here; see `TaskGraphPeerGrant`. */ + readonly issueGrant: (label: string) => Effect.Effect; + readonly setWeight: ( + environmentId: EnvironmentId, + weight: number, + ) => Effect.Effect; + } +>()("t3/taskGraph/TaskGraphPeers") {} + +/** No peers: every node runs on this machine. Used by tests. */ +export const layerNone = Layer.succeed( + TaskGraphPeers, + TaskGraphPeers.of({ + hasPeers: Effect.succeed(false), + candidates: () => Effect.succeed([]), + startNode: () => Effect.fail(new TaskGraphError({ message: "No peer machines are paired." })), + track: () => Effect.void, + interruptNode: () => Effect.void, + completions: Stream.never, + list: Effect.succeed({ peers: [] }), + subscribe: Stream.make({ peers: [] }), + add: () => Effect.fail(new TaskGraphError({ message: "Peers are unavailable." })), + remove: () => Effect.succeed({ peers: [] }), + issueGrant: () => Effect.fail(new TaskGraphError({ message: "Peers are unavailable." })), + setWeight: () => Effect.succeed({ peers: [] }), + }), +); + +const DEFAULT_PEER_WEIGHT = 50; +const tokenSecretName = (environmentId: string) => `task-graph-peer-token:${environmentId}`; + +/** What delivering a peer node's branch produced: its PR, or why only the PR step failed. */ +interface PeerDelivered { + readonly url: string | null; + readonly prError: string | undefined; +} + +const peerError = (message: string, cause?: unknown) => + new TaskGraphError({ message, ...(cause === undefined ? {} : { cause }) }); + +const TERMINAL_THREAD_STATUSES = new Set([ + "completed", + "failed", + "cancelled", + "interrupted", + "rolled_back", +]); + +/** The outcome a finished peer thread implies, before delivery; null while it still runs. */ +function peerThreadOutcome( + thread: Pick, +): "succeeded" | "failed" | "cancelled" | null { + if (thread.activeRunId !== null || !TERMINAL_THREAD_STATUSES.has(thread.status)) return null; + if (thread.status === "completed") return "succeeded"; + return thread.status === "cancelled" || thread.status === "interrupted" ? "cancelled" : "failed"; +} + +const EnvironmentDescriptorJson = Schema.Struct({ + environmentId: EnvironmentId, + label: Schema.String, +}); +const TokenResponse = Schema.Struct({ + access_token: Schema.String, + token_type: Schema.String, +}); +const decodeDescriptor = Schema.decodeUnknownEffect(EnvironmentDescriptorJson); +const decodeToken = Schema.decodeUnknownEffect(TokenResponse); + +const toWsUrl = (httpBaseUrl: string) => { + const url = new URL("/ws", httpBaseUrl); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + url.searchParams.set("orchestrationProtocol", "2"); + return url.toString(); +}; + +const protocolLayer = (httpBaseUrl: string, token: string) => + RpcClient.layerProtocolSocket().pipe( + Layer.provide( + Socket.layerWebSocket(toWsUrl(httpBaseUrl)).pipe( + Layer.provide( + Layer.succeed( + Socket.WebSocketConstructor, + (url, protocols) => + new NodeSocket.NodeWS.WebSocket(url, protocols as string | string[] | undefined, { + headers: { authorization: `Bearer ${token}` }, + }) as unknown as globalThis.WebSocket, + ), + ), + ), + ), + Layer.provide(RpcSerialization.layerJson), + ); + +const makeClient = RpcClient.make(WsRpcGroup); +type PeerClient = Effect.Success; + +interface PeerRow { + readonly environment_id: string; + readonly label: string; + readonly http_base_url: string; + readonly weight: number; + readonly added_at: string; +} + +interface PeerLive { + status: TaskGraphPeer["status"]; + error: string | null; + client: PeerClient | null; + readonly projects: Map; + readonly threads: Map; +} + +interface Tracked { + readonly environmentId: EnvironmentId; + readonly delivery: PeerDelivery; + delivering: boolean; + /** True until the peer accepts the launch; until then a snapshot may not list the thread yet. */ + launching: boolean; +} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const environment = yield* ServerEnvironment.ServerEnvironment; + const projects = yield* ProjectService.ProjectService; + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const localEnvironmentId = yield* environment.getEnvironmentId; + const http = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + + /** Reads a JSON body from a peer during pairing; any failure means the link is unusable. */ + const fetchJson = (request: HttpClientRequest.HttpClientRequest) => + http.execute(request).pipe( + Effect.flatMap((response) => response.json), + Effect.timeout("10 seconds"), + Effect.mapError((cause) => + peerError(`Could not reach ${new URL(request.url).origin}.`, cause), + ), + ); + + const live = new Map(); + const tracked = new Map(); + const connections = yield* FiberMap.make(); + const changes = yield* PubSub.sliding(1); + const completions = yield* Queue.unbounded<{ threadId: ThreadId; outcome: NodeOutcome }>(); + const notify = PubSub.publish(changes, undefined).pipe(Effect.asVoid); + + const readRows = sql` + SELECT environment_id, label, http_base_url, weight, added_at + FROM task_graph_peers ORDER BY added_at + `.pipe(Effect.mapError((cause) => peerError("Could not read peer machines.", cause))); + + const list = readRows.pipe( + Effect.map((rows) => ({ + peers: rows.map((row): TaskGraphPeer => { + const state = live.get(row.environment_id); + return { + environmentId: EnvironmentId.make(row.environment_id), + label: row.label, + httpBaseUrl: row.http_base_url, + weight: row.weight, + status: state?.status ?? "connecting", + error: state?.error ?? null, + addedAt: row.added_at, + }; + }), + })), + ); + + const setStatus = ( + environmentId: string, + status: TaskGraphPeer["status"], + error: string | null, + ) => + Effect.suspend(() => { + const state = live.get(environmentId); + if (state === undefined || (state.status === status && state.error === error)) + return Effect.void; + state.status = status; + state.error = error; + return notify; + }); + + // --- Remote node lifecycle ---------------------------------------------------- + + /** Commits and pushes a finished peer thread's branch there, then reports the node. */ + const finishTracked = (environmentId: string, thread: OrchestrationV2ThreadShell) => + Effect.gen(function* () { + const entry = tracked.get(thread.id); + const state = live.get(environmentId); + const outcome = peerThreadOutcome(thread); + if (entry === undefined || entry.delivering || outcome === null || state?.client == null) + return; + entry.delivering = true; + if (outcome !== "succeeded") { + tracked.delete(thread.id); + yield* Queue.offer(completions, { + threadId: thread.id, + outcome: { type: outcome, error: `The node's run on ${environmentId} ${thread.status}.` }, + }); + return; + } + const summary = + thread.latestVisibleMessage?.role === "assistant" ? thread.latestVisibleMessage.text : null; + const delivered = + thread.worktreePath === null || entry.delivery.action === "none" + ? Option.none() + : Option.some( + yield* state.client[WS_METHODS.gitRunStackedAction]({ + actionId: `task-graph:${thread.id}`, + cwd: thread.worktreePath, + action: entry.delivery.action, + ...(entry.delivery.baseBranch === null + ? {} + : { baseBranch: entry.delivery.baseBranch }), + threadId: thread.id, + projectId: thread.projectId, + }).pipe( + Stream.runCollect, + Effect.flatMap((events): Effect.Effect => { + for (const event of events) { + // As on this machine, a failed PR step alone leaves the node succeeded: + // its work is already committed and pushed for dependents. + if ( + event.kind === "action_failed" && + event.phase === "pr" && + entry.delivery.action === "commit_push_pr" + ) { + return Effect.succeed({ url: null, prError: event.message }); + } + if (event.kind === "action_failed") + return Effect.fail(peerError(event.message)); + if (event.kind === "action_finished") { + return Effect.succeed({ + url: event.result.pr.url ?? null, + prError: undefined, + }); + } + } + return Effect.fail(peerError("The peer did not report how delivery went.")); + }), + ), + ); + tracked.delete(thread.id); + // Settle the finished thread there, as a local node's is. Servers without + // settlement refuse the command; the node still counts as succeeded. + yield* state.client[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]({ + type: "thread.settle", + commandId: CommandId.make(`task-graph-settle:${thread.id}`), + threadId: thread.id, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not settle a peer task graph node's thread", { + environmentId, + threadId: thread.id, + cause: Cause.pretty(cause), + }), + ), + ); + yield* Queue.offer(completions, { + threadId: thread.id, + outcome: { + type: "succeeded", + summary, + branch: thread.branch, + worktreePath: thread.worktreePath, + ...(entry.delivery.action === "commit_push_pr" && Option.isSome(delivered) + ? delivered.value.prError === undefined + ? { pullRequestUrl: delivered.value.url } + : { pullRequestError: delivered.value.prError } + : {}), + }, + }); + }).pipe( + Effect.catchCause((cause) => { + tracked.delete(thread.id); + return Queue.offer(completions, { + threadId: thread.id, + outcome: { + type: "failed", + error: `Could not push the node's branch from the peer: ${Cause.pretty(cause)}`, + }, + }).pipe(Effect.asVoid); + }), + ); + + const handleShellItem = ( + environmentId: string, + state: PeerLive, + item: OrchestrationV2ShellStreamItem, + ) => + Effect.gen(function* () { + const changed: OrchestrationV2ThreadShell[] = []; + switch (item.kind) { + case "snapshot": { + // An enrichment refresh only patches repository identity; it never replaces state. + if (item.resolvedRepositoryIdentityRoots !== undefined) { + for (const project of item.snapshot.projects) { + const known = state.projects.get(project.id); + if (known !== undefined && known.workspaceRoot === project.workspaceRoot) { + state.projects.set(project.id, { + ...known, + repositoryIdentity: project.repositoryIdentity, + }); + } + } + break; + } + state.projects.clear(); + state.threads.clear(); + for (const project of item.snapshot.projects) state.projects.set(project.id, project); + for (const thread of [...item.snapshot.threads, ...item.snapshot.archivedThreads]) { + state.threads.set(thread.id, thread); + changed.push(thread); + } + // A tracked thread the full snapshot lacks was deleted, or never created before a + // restart; no removal event will come, so fail its node rather than wait forever. + for (const [threadId, entry] of tracked) { + if ( + entry.environmentId !== environmentId || + entry.launching || + state.threads.has(threadId) + ) { + continue; + } + tracked.delete(threadId); + yield* Queue.offer(completions, { + threadId, + outcome: { type: "failed", error: "The node's thread is missing on the peer." }, + }); + } + break; + } + case "project.updated": + state.projects.set(item.project.id, item.project); + break; + case "project.removed": + state.projects.delete(item.projectId); + break; + case "thread.updated": + state.threads.set(item.thread.id, item.thread); + changed.push(item.thread); + break; + case "thread.removed": { + state.threads.delete(item.threadId); + const entry = tracked.get(item.threadId); + if (entry !== undefined && entry.environmentId === environmentId) { + tracked.delete(item.threadId); + yield* Queue.offer(completions, { + threadId: item.threadId, + outcome: { type: "failed", error: "The node's thread was deleted on the peer." }, + }); + } + break; + } + default: + break; + } + for (const thread of changed) { + if (tracked.get(thread.id)?.environmentId === environmentId) { + // Delivery waits on the peer's git work; do not hold up the shell stream. + yield* finishTracked(environmentId, thread).pipe(Effect.forkDetach); + } + } + }); + + /** One connection to a peer, until it drops. */ + const connectOnce = (row: PeerRow, token: string, state: PeerLive) => + Effect.gen(function* () { + const client = yield* makeClient; + const config = yield* client[WS_METHODS.serverGetConfig]({}); + if (config.environment.environmentId !== row.environment_id) { + return yield* peerError( + `The machine at ${row.http_base_url} is now a different environment. Pair it again.`, + ); + } + state.client = client; + yield* setStatus(row.environment_id, "connected", null); + yield* client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({}).pipe( + Stream.runForEach((item) => handleShellItem(row.environment_id, state, item)), + ); + }).pipe( + Effect.scoped, + Effect.provide(protocolLayer(row.http_base_url, token)), + Effect.ensuring(Effect.sync(() => (state.client = null))), + ); + + const connect = (row: PeerRow) => + Effect.gen(function* () { + const token = yield* secrets.get(tokenSecretName(row.environment_id)); + if (Option.isNone(token)) { + return yield* setStatus( + row.environment_id, + "unauthorized", + "The peer's credential is missing. Pair it again.", + ); + } + const state = live.get(row.environment_id)!; + const tokenText = new TextDecoder().decode(token.value); + yield* connectOnce(row, tokenText, state).pipe( + Effect.catchCause((cause) => { + const message = Cause.pretty(cause); + return setStatus( + row.environment_id, + /\b401\b|unauthori[sz]ed/i.test(message) ? "unauthorized" : "unreachable", + message.split("\n")[0] ?? message, + ); + }), + Effect.repeat(Schedule.spaced("15 seconds")), + ); + }).pipe(Effect.ignoreCause); + + const startConnection = (row: PeerRow) => + Effect.gen(function* () { + live.set(row.environment_id, { + status: "connecting", + error: null, + client: null, + projects: new Map(), + threads: new Map(), + }); + yield* FiberMap.run(connections, row.environment_id, connect(row)); + }); + + yield* readRows.pipe( + Effect.flatMap((rows) => Effect.forEach(rows, startConnection, { discard: true })), + Effect.catchCause((cause) => + Effect.logWarning("Could not start task graph peer connections", { + cause: Cause.pretty(cause), + }), + ), + ); + + // --- Service ------------------------------------------------------------------ + + const repositoryKeyOf = (projectId: ProjectId) => + projects.getShell(projectId).pipe( + Effect.map((shell) => + Option.flatMap(shell, (project) => + Option.fromNullishOr(project.repositoryIdentity).pipe( + Option.map(repositoryGroupingKeyOf), + ), + ), + ), + Effect.orElseSucceed(() => Option.none()), + ); + + const peerProjectFor = (state: PeerLive, key: string) => + [...state.projects.values()].find( + (project) => + project.repositoryIdentity != null && + repositoryGroupingKeyOf(project.repositoryIdentity) === key, + ); + + const candidates: TaskGraphPeers["Service"]["candidates"] = ({ projectId }) => + Effect.gen(function* () { + const key = yield* repositoryKeyOf(projectId); + if (Option.isNone(key)) return []; + const rows = yield* readRows.pipe(Effect.orElseSucceed(() => [])); + const usable = rows.flatMap((row) => { + const state = live.get(row.environment_id); + const running = [...tracked.values()].filter( + (entry) => entry.environmentId === row.environment_id, + ).length; + return state?.client != null && peerProjectFor(state, key.value) !== undefined + ? [{ row, client: state.client, running }] + : []; + }); + return yield* Effect.forEach( + usable, + ({ row, client, running }) => + client[WS_METHODS.serverGetHostResources]({}).pipe( + Effect.timeout("5 seconds"), + Effect.orElseSucceed(() => null), + Effect.flatMap((resources) => + Clock.currentTimeMillis.pipe( + Effect.map((receivedAt) => ({ + environmentId: EnvironmentId.make(row.environment_id), + resources, + receivedAt, + weight: row.weight, + running, + })), + ), + ), + ), + { concurrency: "unbounded" }, + ); + }); + + const startNode: TaskGraphPeers["Service"]["startNode"] = (input) => + Effect.gen(function* () { + const state = live.get(input.environmentId); + const client = state?.client; + if (state === undefined || client == null) { + return yield* peerError(`The peer ${input.environmentId} is not connected.`); + } + const key = yield* repositoryKeyOf(input.graph.projectId); + const project = Option.isNone(key) ? undefined : peerProjectFor(state, key.value); + if (project === undefined) { + return yield* peerError("The peer has no project for this repository."); + } + const entry: Tracked = { + environmentId: input.environmentId, + delivery: input.delivery, + delivering: false, + launching: true, + }; + tracked.set(input.threadId, entry); + yield* client[ORCHESTRATION_V2_WS_METHODS.launchThread]({ + commandId: CommandId.make(`task-graph-launch:${input.threadId}`), + creationSource: "server", + threadId: input.threadId, + projectId: project.id, + title: input.node.title, + modelSelection: input.modelSelection, + runtimeMode: input.runtimeMode, + interactionMode: "default", + workspaceStrategy: input.workspaceStrategy, + initialMessage: { text: input.prompt, attachments: [] }, + }).pipe( + Effect.tapError(() => Effect.sync(() => tracked.delete(input.threadId))), + Effect.mapError((cause) => peerError("The peer could not start the node.", cause)), + ); + entry.launching = false; + }); + + const track: TaskGraphPeers["Service"]["track"] = ({ environmentId, threadId, delivery }) => + Effect.gen(function* () { + tracked.set(threadId, { environmentId, delivery, delivering: false, launching: false }); + const thread = live.get(environmentId)?.threads.get(threadId); + if (thread !== undefined) yield* finishTracked(environmentId, thread).pipe(Effect.forkDetach); + }); + + const interruptNode: TaskGraphPeers["Service"]["interruptNode"] = ({ environmentId, threadId }) => + Effect.gen(function* () { + tracked.delete(threadId); + const state = live.get(environmentId); + const runId = state?.threads.get(threadId)?.activeRunId; + if (state?.client == null || runId == null) return; + yield* state.client[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]({ + type: "run.interrupt", + commandId: CommandId.make(`task-graph-stop:${threadId}`), + threadId, + runId, + reason: "The task graph branch was cancelled.", + }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not stop a peer task graph node", { + environmentId, + threadId, + cause: Cause.pretty(cause), + }), + ), + ); + + const add: TaskGraphPeers["Service"]["add"] = (input) => + Effect.gen(function* () { + const target = yield* Effect.try({ + try: () => resolveRemotePairingTarget({ pairingUrl: input.pairingUrl }), + catch: (cause) => peerError("That is not a pairing link.", cause), + }); + const descriptor = yield* fetchJson( + HttpClientRequest.get( + new URL("/.well-known/t3/environment", target.httpBaseUrl).toString(), + ), + ).pipe( + Effect.flatMap((body) => + decodeDescriptor(body).pipe( + Effect.mapError((cause) => + peerError("The link does not point at a T3 Code server.", cause), + ), + ), + ), + ); + if (descriptor.environmentId === localEnvironmentId) { + return yield* peerError("That link points at this machine."); + } + const label = input.label ?? (descriptor.label.trim() || target.httpBaseUrl); + const token = yield* fetchJson( + HttpClientRequest.post(new URL("/oauth/token", target.httpBaseUrl).toString()).pipe( + HttpClientRequest.bodyUrlParams({ + grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", + subject_token: target.credential, + subject_token_type: "urn:t3:params:oauth:token-type:environment-bootstrap", + requested_token_type: "urn:ietf:params:oauth:token-type:access_token", + scope: TASK_GRAPH_PEER_SCOPES.join(" "), + client_label: "T3 Code task graphs", + client_device_type: "bot", + }), + ), + ).pipe( + Effect.flatMap((body) => + decodeToken(body).pipe( + Effect.mapError((cause) => peerError("The peer refused the link.", cause)), + ), + ), + ); + if (token.token_type.toLowerCase() !== "bearer") { + return yield* peerError("The peer issued a credential this server cannot use."); + } + yield* secrets + .set( + tokenSecretName(descriptor.environmentId), + new TextEncoder().encode(token.access_token), + ) + .pipe( + Effect.mapError((cause) => peerError("Could not store the peer's credential.", cause)), + ); + const addedAt = DateTime.formatIso(yield* DateTime.now); + const row: PeerRow = { + environment_id: descriptor.environmentId, + label, + http_base_url: target.httpBaseUrl, + weight: DEFAULT_PEER_WEIGHT, + added_at: addedAt, + }; + yield* sql` + INSERT INTO task_graph_peers (environment_id, label, http_base_url, weight, added_at) + VALUES (${row.environment_id}, ${row.label}, ${row.http_base_url}, ${row.weight}, ${row.added_at}) + ON CONFLICT (environment_id) DO UPDATE SET + label = excluded.label, + http_base_url = excluded.http_base_url + `.pipe(Effect.mapError((cause) => peerError("Could not save the peer.", cause))); + yield* startConnection(row); + yield* notify; + return yield* list; + }); + + const remove: TaskGraphPeers["Service"]["remove"] = (environmentId) => + Effect.gen(function* () { + yield* FiberMap.remove(connections, environmentId); + live.delete(environmentId); + yield* sql`DELETE FROM task_graph_peers WHERE environment_id = ${environmentId}`.pipe( + Effect.mapError((cause) => peerError("Could not remove the peer.", cause)), + ); + // Forgetting the token is what unpairs on this side; the peer lists the + // session under its connected clients until someone revokes it there. + yield* secrets.remove(tokenSecretName(environmentId)).pipe(Effect.ignore); + for (const [threadId, entry] of tracked) { + if (entry.environmentId !== environmentId) continue; + tracked.delete(threadId); + yield* Queue.offer(completions, { + threadId, + outcome: { type: "failed", error: "The peer running this node was removed." }, + }); + } + yield* notify; + return yield* list; + }); + + const setWeight: TaskGraphPeers["Service"]["setWeight"] = (environmentId, weight) => + sql`UPDATE task_graph_peers SET weight = ${weight} WHERE environment_id = ${environmentId}`.pipe( + Effect.mapError((cause) => peerError("Could not update the peer.", cause)), + Effect.andThen(notify), + Effect.andThen(list), + ); + + const issueGrant: TaskGraphPeers["Service"]["issueGrant"] = (label) => + auth + .createPairingLink({ + scopes: TASK_GRAPH_PEER_SCOPES, + subject: "task-graph-peer", + ttl: Duration.minutes(5), + label: `Task graphs: ${label}`, + }) + .pipe( + Effect.map((link) => ({ + credential: link.credential, + expiresAt: DateTime.formatIso(link.expiresAt), + })), + Effect.mapError((cause) => peerError("Could not issue a pairing grant.", cause)), + ); + + const subscribe = Stream.unwrap( + Effect.gen(function* () { + const subscription = yield* PubSub.subscribe(changes); + return Stream.concat( + Stream.fromEffect(list), + Stream.fromSubscription(subscription).pipe(Stream.mapEffect(() => list)), + ); + }), + ); + + return TaskGraphPeers.of({ + hasPeers: readRows.pipe( + Effect.map((rows) => rows.length > 0), + Effect.orElseSucceed(() => false), + ), + candidates, + startNode, + track, + interruptNode, + completions: Stream.fromQueue(completions), + list, + subscribe, + add, + remove, + issueGrant, + setWeight, + }); +}); + +export const layer = Layer.effect(TaskGraphPeers, make).pipe(Layer.provide(FetchHttpClient.layer)); diff --git a/apps/server/src/taskGraph/TaskGraphService.test.ts b/apps/server/src/taskGraph/TaskGraphService.test.ts new file mode 100644 index 000000000000..6bb273817d56 --- /dev/null +++ b/apps/server/src/taskGraph/TaskGraphService.test.ts @@ -0,0 +1,639 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { assert, it } from "@effect/vitest"; +import { + EnvironmentId, + type GitRunStackedActionInput, + type OrchestrationV2DomainEvent, + type OrchestrationV2ServerCommand, + type OrchestrationV2ThreadProjection, + ThreadId, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as GitWorkflow from "../git/GitWorkflowService.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; +import * as Scheduler from "../scheduling/Scheduler.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as TaskGraphPeers from "./TaskGraphPeers.ts"; +import * as TaskGraphService from "./TaskGraphService.ts"; + +const PARENT = ThreadId.make("thread-parent"); + +interface Harness { + readonly launches: Queue.Queue; + readonly gitActions: Array; + readonly interrupted: Array; + /** Node threads the service settled, in order. */ + readonly settled: Queue.Queue; + /** Commands the service dispatched to threads, such as arming usage limit recovery. */ + readonly dispatched: Array; + /** Replaces a node thread's projection, then emits a run event with `status`. */ + readonly setThread: ( + threadId: ThreadId, + projection: OrchestrationV2ThreadProjection, + status: string, + ) => Effect.Effect; + readonly report: Deferred.Deferred; + /** + * Sets a node thread's latest run, completed by default, with a reply on a + * branch, then emits the run event. + */ + readonly finish: ( + launch: ThreadLaunch.ThreadLaunchInput, + result: { readonly reply: string; readonly branch: string; readonly status?: string }, + ) => Effect.Effect; + /** Makes the next git deliveries fail in `phase`, or succeed again with null. */ + readonly failGitIn: (phase: "commit" | "push" | "pr" | null) => void; +} + +const projection = (input: { + readonly id: ThreadId; + readonly branch: string | null; + readonly worktreePath: string | null; + readonly runStatus?: string; + readonly reply?: string; +}) => + ({ + thread: { + id: input.id, + projectId: "project-1", + modelSelection: { instanceId: "codex", model: "gpt-5" }, + runtimeMode: "full-access", + branch: input.branch, + worktreePath: input.worktreePath, + }, + runs: input.runStatus === undefined ? [] : [{ status: input.runStatus, ordinal: 1 }], + messages: input.reply === undefined ? [] : [{ role: "assistant", text: input.reply }], + turnItems: [], + }) as unknown as OrchestrationV2ThreadProjection; + +/** Builds the service against in-memory SQLite and recording fakes for everything it drives. */ +const withService = ( + body: ( + service: TaskGraphService.TaskGraphService["Service"], + harness: Harness, + ) => Effect.Effect, + peers: Layer.Layer = TaskGraphPeers.layerNone, +) => + Effect.gen(function* () { + const launches = yield* Queue.unbounded(); + const events = yield* Queue.unbounded(); + const report = yield* Deferred.make(); + const projections = new Map([ + [PARENT, projection({ id: PARENT, branch: "main", worktreePath: null })], + ]); + const gitActions: Array = []; + const interrupted: Array = []; + const settled = yield* Queue.unbounded(); + const dispatched: Array = []; + let gitFailurePhase: "commit" | "push" | "pr" | null = null; + + const harness: Harness = { + failGitIn: (phase) => { + gitFailurePhase = phase; + }, + launches, + gitActions, + interrupted, + settled, + dispatched, + setThread: (threadId, next, status) => + Effect.gen(function* () { + projections.set(threadId, next); + yield* Queue.offer(events, { + type: "run.updated", + threadId, + payload: { status }, + } as unknown as OrchestrationV2DomainEvent); + }), + report, + finish: (launch, result) => + Effect.gen(function* () { + const threadId = launch.threadId!; + projections.set( + threadId, + projection({ + id: threadId, + branch: result.branch, + worktreePath: `/worktrees/${result.branch}`, + runStatus: result.status ?? "completed", + reply: result.reply, + }), + ); + yield* Queue.offer(events, { + type: "run.updated", + threadId, + payload: { status: result.status ?? "completed" }, + } as unknown as OrchestrationV2DomainEvent); + }), + }; + + const layer = TaskGraphService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ThreadLaunch.ThreadLaunchService)({ + launch: (input) => + Queue.offer(launches, input).pipe( + Effect.as({ + threadId: input.threadId!, + projection: projection({ id: input.threadId!, branch: null, worktreePath: null }), + resumed: false, + }), + ), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadProjection: (threadId) => + Effect.suspend(() => { + const found = projections.get(threadId); + return found === undefined + ? Effect.die(`no thread ${threadId}`) + : Effect.succeed(found); + }), + streamDomainEvents: Stream.fromQueue(events), + sendToThread: (input) => + Deferred.succeed(report, input.text).pipe(Effect.as({} as never)), + dispatch: (command) => + Effect.sync(() => { + dispatched.push(command); + return {} as never; + }), + settleThread: (input) => + Queue.offer(settled, input.threadId).pipe(Effect.as({ sequence: 0 })), + interruptThread: (input) => + Effect.sync(() => { + interrupted.push(input.threadId); + return { type: "no_active_run" as const }; + }), + }), + Layer.mock(GitWorkflow.GitWorkflowService)({ + runStackedAction: (input, options) => + Effect.suspend(() => { + gitActions.push(input); + const phase = gitFailurePhase; + if (phase !== null) { + return ( + options?.progressReporter?.publish({ + kind: "action_failed", + actionId: input.actionId, + cwd: input.cwd, + action: input.action, + phase, + message: `${phase} failed`, + } as never) ?? Effect.void + ).pipe(Effect.andThen(Effect.fail(new Error(`${phase} failed`) as never))); + } + return Effect.succeed({ + pr: + input.action === "commit_push_pr" + ? { status: "created", url: `https://pr/${input.cwd}` } + : {}, + } as never); + }), + }), + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), + Layer.mock(HostResources.HostResources)({ + read: Effect.succeed({ + sampledAt: 0, + cpuUtilization: 0.1, + cpuCount: 8, + availableMemoryBytes: 8e9, + totalMemoryBytes: 16e9, + }), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("env-local")), + }), + // Linking the created PR is best effort; these fakes make it log and move on. + Layer.mock(Orchestrator.OrchestratorV2)({}), + Layer.mock(ProjectService.ProjectService)({}), + ServerSettings.layerTest({}), + peers, + Scheduler.layer, + NodeCrypto.layer, + ), + ), + ); + return yield* TaskGraphService.TaskGraphService.pipe( + Effect.flatMap((service) => body(service, harness)), + Effect.provide(layer), + ); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)); + +const node = (key: string, dependsOn: ReadonlyArray = []) => ({ + key, + title: key, + prompt: `Do ${key}.`, + dependsOn, +}); + +it.effect("fans out, merges, opens a PR at the end, and reports to the proposing thread", () => + withService((service, harness) => + Effect.gen(function* () { + yield* service.create({ + threadId: PARENT, + title: "Security audit", + nodes: [node("a"), node("b", ["a"]), node("c", ["a"]), node("d", ["b", "c"])], + run: true, + }); + + const a = yield* Queue.take(harness.launches); + assert.equal(a.title, "a"); + // The node's thread does not exist yet; reuseExistingThread would make the launcher + // look for it and fail every launch. + assert.isUndefined(a.reuseExistingThread); + assert.isDefined(a.threadId); + assert.deepEqual(a.workspaceStrategy, { type: "worktree", baseRef: "main" }); + yield* harness.finish(a, { reply: "a found two issues", branch: "t3/a" }); + + // Both branches of the fan-out start from a's branch once a succeeds. + const fanOut = [yield* Queue.take(harness.launches), yield* Queue.take(harness.launches)]; + assert.deepEqual(fanOut.map((launch) => launch.title).toSorted(), ["b", "c"]); + for (const launch of fanOut) { + assert.deepEqual(launch.workspaceStrategy, { type: "worktree", baseRef: "t3/a" }); + assert.include(launch.initialMessage!.text, "a found two issues"); + yield* harness.finish(launch, { + reply: `${launch.title} fixed`, + branch: `t3/${launch.title}`, + }); + } + + const d = yield* Queue.take(harness.launches); + assert.equal(d.title, "d"); + assert.deepEqual(d.workspaceStrategy, { type: "worktree", baseRef: "t3/b" }); + assert.include(d.initialMessage!.text, "- t3/c (from 'c')"); + assert.include(d.initialMessage!.text, "b fixed"); + assert.include(d.initialMessage!.text, "c fixed"); + yield* harness.finish(d, { reply: "merged", branch: "t3/d" }); + + const report = yield* Deferred.await(harness.report); + assert.include(report, 'Task graph "Security audit" finished: succeeded.'); + // Each succeeded node's thread is settled, so finished work leaves the active list. + const settled = yield* Effect.forEach([a, ...fanOut, d], () => Queue.take(harness.settled)); + assert.deepEqual( + settled.toSorted(), + [a, ...fanOut, d].map((launch) => launch.threadId!).toSorted(), + ); + assert.include(report, "PR: https://pr//worktrees/t3/d"); + // Only the branch end opens a PR, so it targets the graph base and carries all the work. + assert.equal( + harness.gitActions.find((action) => action.action === "commit_push_pr")?.baseBranch, + "main", + ); + // Inner nodes commit so dependents can branch from them; only the end opens a PR. + assert.deepEqual( + harness.gitActions.map((action) => [action.cwd, action.action]), + [ + ["/worktrees/t3/a", "commit"], + ...fanOut + .map((launch) => [`/worktrees/t3/${launch.title}`, "commit"]) + .toSorted(([left], [right]) => left!.localeCompare(right!)), + ["/worktrees/t3/d", "commit_push_pr"], + ].toSorted(([left], [right]) => left!.localeCompare(right!)), + ); + }), + ), +); + +it.effect("cutting off a running branch stops its thread and skips what depends on it", () => + withService((service, harness) => + Effect.gen(function* () { + const graph = yield* service.create({ + threadId: PARENT, + title: "Features", + nodes: [node("a"), node("b", ["a"])], + run: true, + }); + const a = yield* Queue.take(harness.launches); + + const edited = yield* service.edit(graph.id, [{ type: "cancel_branch", key: "a" }]); + + assert.deepEqual(harness.interrupted, [a.threadId]); + assert.deepEqual( + edited.nodes.map((graphNode) => graphNode.status), + ["cancelled", "cancelled"], + ); + assert.equal(edited.status, "cancelled"); + assert.include(yield* Deferred.await(harness.report), "finished: cancelled"); + }), + ), +); + +it.effect("a draft waits for run, and edits before it starts change what launches", () => + withService((service, harness) => + Effect.gen(function* () { + const draft = yield* service.create({ + threadId: PARENT, + title: "Plan", + nodes: [node("a")], + run: false, + }); + assert.equal(draft.status, "draft"); + assert.equal(yield* Queue.size(harness.launches), 0); + + yield* service.edit(draft.id, [ + { type: "update_node", key: "a", prompt: "Do a differently." }, + { type: "add_node", node: node("b") }, + ]); + yield* service.run(draft.id); + + const launched = [yield* Queue.take(harness.launches), yield* Queue.take(harness.launches)]; + assert.deepEqual(launched.map((launch) => launch.title).toSorted(), ["a", "b"]); + assert.include( + launched.find((launch) => launch.title === "a")!.initialMessage!.text, + "Do a differently.", + ); + }), + ), +); + +it.effect("places a node on a peer with more free capacity and pushes local branches for it", () => + Effect.gen(function* () { + const started = yield* Queue.unbounded(); + const peers = Layer.succeed( + TaskGraphPeers.TaskGraphPeers, + TaskGraphPeers.TaskGraphPeers.of({ + hasPeers: Effect.succeed(true), + // A much bigger idle machine: it wins the load-balancing score. + candidates: () => + Effect.succeed([ + { + environmentId: EnvironmentId.make("env-peer"), + resources: { + sampledAt: 0, + cpuUtilization: 0, + cpuCount: 64, + availableMemoryBytes: 100e9, + totalMemoryBytes: 128e9, + }, + // it.effect runs on a test clock that starts at 0. + receivedAt: 0, + weight: 50, + running: 0, + }, + ]), + startNode: (input) => Queue.offer(started, input).pipe(Effect.asVoid), + track: () => Effect.void, + interruptNode: () => Effect.void, + completions: Stream.never, + list: Effect.succeed({ peers: [] }), + subscribe: Stream.never, + add: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + issueGrant: () => Effect.die("unused"), + setWeight: () => Effect.die("unused"), + }), + ); + yield* withService( + (service) => + Effect.gen(function* () { + yield* service.create({ + threadId: PARENT, + title: "Spread", + nodes: [node("a")], + run: true, + }); + const start = yield* Queue.take(started); + assert.equal(start.environmentId, "env-peer"); + assert.deepEqual(start.workspaceStrategy, { + type: "worktree", + baseRef: "main", + startFromOrigin: true, + }); + // A branch end opens its PR from the peer. + assert.deepEqual(start.delivery, { action: "commit_push_pr", baseBranch: "main" }); + }), + peers, + ); + }), +); + +it.effect("follows a failed node that someone continues in its own thread", () => + withService((service, harness) => + Effect.gen(function* () { + const graph = yield* service.create({ + threadId: PARENT, + title: "Roadmap", + nodes: [node("a"), node("b", ["a"])], + run: true, + }); + const a = yield* Queue.take(harness.launches); + yield* harness.finish(a, { reply: "crashed", branch: "t3/a", status: "failed" }); + assert.include(yield* Deferred.await(harness.report), "finished: failed"); + // A failed node's thread stays active so someone can pick it up. + assert.equal(yield* Queue.size(harness.settled), 0); + + // The user sends a follow-up in a's thread and that run succeeds. + yield* harness.finish(a, { reply: "", branch: "t3/a", status: "running" }); + yield* harness.finish(a, { reply: "fixed it", branch: "t3/a" }); + + const b = yield* Queue.take(harness.launches); + assert.equal(b.title, "b"); + assert.include(b.initialMessage!.text, "fixed it"); + const resumed = yield* service.get(graph.id); + assert.equal(resumed.status, "running"); + assert.deepEqual( + resumed.nodes.map((graphNode) => graphNode.status), + ["succeeded", "running"], + ); + }), + ), +); + +it.effect("stacks PRs, continues a worktree in place, and never commits project-folder work", () => + withService((service, harness) => + Effect.gen(function* () { + yield* service.create({ + threadId: PARENT, + title: "Stack", + nodes: [ + { ...node("review"), workspace: "root" }, + { ...node("a", ["review"]), pullRequest: true }, + { ...node("a-tests", ["a"]), workspace: "dependency" }, + { ...node("b", ["a-tests"]), pullRequest: true }, + ], + run: true, + }); + + const review = yield* Queue.take(harness.launches); + assert.deepEqual(review.workspaceStrategy, { type: "root" }); + yield* harness.finish(review, { reply: "read it all", branch: "main" }); + + const a = yield* Queue.take(harness.launches); + assert.deepEqual(a.workspaceStrategy, { type: "worktree", baseRef: "main" }); + yield* harness.finish(a, { reply: "layer one", branch: "t3/a" }); + + const aTests = yield* Queue.take(harness.launches); + assert.deepEqual(aTests.workspaceStrategy, { + type: "existing_worktree", + worktreePath: "/worktrees/t3/a", + branch: "t3/a", + }); + yield* harness.finish(aTests, { reply: "tests added", branch: "t3/a" }); + + const b = yield* Queue.take(harness.launches); + yield* harness.finish(b, { reply: "layer two", branch: "t3/b" }); + yield* Deferred.await(harness.report); + + // The review committed nothing; each layer's PR targets the layer below it. + assert.deepEqual( + harness.gitActions.map((action) => [action.cwd, action.action, action.baseBranch]), + [ + ["/worktrees/t3/a", "commit_push_pr", "main"], + // a-tests continues a's branch, which has a PR, so it pushes for that PR to get it. + ["/worktrees/t3/a", "commit_push", undefined], + ["/worktrees/t3/b", "commit_push_pr", "t3/a"], + ], + ); + }), + ), +); + +/** A thread whose only run stopped on a usage limit that resets at `resetAt`. */ +const limitedThread = (threadId: ThreadId, resetAt: string) => + ({ + ...projection({ id: threadId, branch: "t3/a", worktreePath: "/worktrees/t3/a" }), + runs: [ + { + id: "run-1", + status: "failed", + ordinal: 1, + rootNodeId: "node-1", + startedAt: DateTime.makeUnsafe(0), + completedAt: DateTime.makeUnsafe(1_000), + }, + ], + turnItems: [ + { + id: "item-1", + type: "error", + status: "failed", + runId: "run-1", + nodeId: "node-1", + ordinal: 1, + updatedAt: DateTime.makeUnsafe(1_000), + failure: { class: "usage_limit", message: "Limit reached", resetAt }, + }, + ], + }) as unknown as OrchestrationV2ThreadProjection; + +it.effect("holds a node stopped by a usage limit and continues it on its thread at the reset", () => + withService((service, harness) => + Effect.gen(function* () { + const graph = yield* service.create({ + threadId: PARENT, + title: "Limits", + nodes: [node("a"), node("b", ["a"])], + run: true, + }); + const a = yield* Queue.take(harness.launches); + const resetAt = "2026-10-10T05:00:00.000Z"; + yield* harness.setThread(a.threadId!, limitedThread(a.threadId!, resetAt), "failed"); + + // The arm command is the last thing the service does for the limited run. + while (harness.dispatched.length === 0) yield* Effect.yieldNow; + assert.deepEqual(harness.dispatched[0], { + type: "thread.metadata.update", + commandId: `task-graph-limit-arm:${a.threadId}:run-1`, + threadId: a.threadId, + limitRecovery: { runId: "run-1", resetAt, autoResume: true }, + } as never); + const waiting = (yield* service.get(graph.id)).nodes[0]!; + assert.deepEqual( + [waiting.status, waiting.waitUntil, waiting.waitReason], + ["waiting", resetAt, "usage_limit"], + ); + + // At the reset, recovery starts a new run on the same thread, which then succeeds. + yield* harness.setThread(a.threadId!, limitedThread(a.threadId!, resetAt), "running"); + yield* harness.finish(a, { reply: "done after reset", branch: "t3/a" }); + const b = yield* Queue.take(harness.launches); + assert.include(b.initialMessage!.text, "done after reset"); + }), + ), +); + +it.effect("waits for a node's start time", () => + withService((service, harness) => + Effect.gen(function* () { + const graph = yield* service.create({ + threadId: PARENT, + title: "Overnight", + nodes: [{ ...node("a"), startAt: "1970-01-01T00:01:00.000Z" }], + run: true, + }); + assert.deepEqual( + [(yield* service.get(graph.id)).nodes[0]!.status, yield* Queue.size(harness.launches)], + ["waiting", 0], + ); + yield* TestClock.adjust("2 minutes"); + const a = yield* Queue.take(harness.launches); + assert.equal(a.title, "a"); + }), + ), +); + +it.effect("only a failed PR step leaves a node succeeded; a failed commit fails it", () => + withService((service, harness) => + Effect.gen(function* () { + const graph = yield* service.create({ + threadId: PARENT, + title: "Delivery", + nodes: [node("a"), node("b")], + run: true, + }); + const first = yield* Queue.take(harness.launches); + const second = yield* Queue.take(harness.launches); + const [a, b] = first.title === "a" ? [first, second] : [second, first]; + + harness.failGitIn("pr"); + yield* harness.finish(a, { reply: "a done", branch: "t3/a" }); + yield* Queue.take(harness.settled); + harness.failGitIn("commit"); + yield* harness.finish(b, { reply: "b done", branch: "t3/b" }); + assert.include(yield* Deferred.await(harness.report), "finished: failed"); + + const ended = yield* service.get(graph.id); + const byKey = new Map(ended.nodes.map((entry) => [entry.key, entry])); + assert.equal(byKey.get("a")?.status, "succeeded"); + assert.deepEqual(byKey.get("a")?.pullRequestResult, { + status: "failed", + url: null, + error: "pr failed", + }); + assert.equal(byKey.get("b")?.status, "failed"); + assert.include(byKey.get("b")?.error ?? "", "commit failed"); + assert.isNotNull(byKey.get("b")?.completedAt); + }), + ), +); + +it.effect("delivers a node once when its completion arrives twice", () => + withService((service, harness) => + Effect.gen(function* () { + yield* service.create({ threadId: PARENT, title: "Once", nodes: [node("a")], run: true }); + const a = yield* Queue.take(harness.launches); + // Two terminal events for the same run, handled concurrently. + yield* harness.finish(a, { reply: "a done", branch: "t3/a" }); + yield* harness.finish(a, { reply: "a done", branch: "t3/a" }); + assert.include(yield* Deferred.await(harness.report), "finished: succeeded"); + assert.equal(yield* Queue.take(harness.settled), a.threadId); + assert.deepEqual( + harness.gitActions.map((action) => action.cwd), + ["/worktrees/t3/a"], + ); + }), + ), +); diff --git a/apps/server/src/taskGraph/TaskGraphService.ts b/apps/server/src/taskGraph/TaskGraphService.ts new file mode 100644 index 000000000000..1f59a94df183 --- /dev/null +++ b/apps/server/src/taskGraph/TaskGraphService.ts @@ -0,0 +1,1219 @@ +import { + CommandId, + EnvironmentId, + MessageId, + TaskGraph, + TaskGraphError, + TaskGraphId, + ThreadId, + type HostResourcesSnapshot, + type ServerProvider, + type TaskGraphCreateInput, + type TaskGraphEdit, + type TaskGraphListResult, + type TaskGraphNode, +} from "@t3tools/contracts"; +import { + applyTaskGraphEdits, + buildTaskGraphNodePrompt, + deriveTaskGraphStatus, + isActiveTaskGraphNodeStatus, + isTerminalTaskGraphNodeStatus, + newTaskGraphNode, + readyTaskGraphNodes, + resumeTaskGraphNode, + skipUnreachableTaskGraphNodes, + taskGraphNodeOpensPullRequest, + taskGraphNodeSharesPullRequestBranch, + taskGraphNodeSummary, + taskGraphPullRequestBase, + validateTaskGraphNodes, +} from "@t3tools/shared/taskGraph"; +import { chooseLoadBalancedEnvironment } from "@t3tools/shared/loadBalancing"; +import { + latestRootProviderFailure, + usageLimitBlockedRun, +} from "@t3tools/shared/orchestrationV2ThreadError"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SqlClient from "effect/sql/SqlClient"; + +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as GitWorkflow from "../git/GitWorkflowService.ts"; +import { linkCreatedPullRequest } from "../git/linkCreatedPullRequest.ts"; +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ThreadLaunch from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as HostResources from "../resourceTelemetry/HostResources.ts"; +import * as Scheduler from "../scheduling/Scheduler.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as TaskGraphPeers from "./TaskGraphPeers.ts"; + +/** + * Runs task graphs: plans of agent tasks with dependencies, where each node is + * its own thread in its own worktree. The orchestrator stays unaware of graphs; + * this service launches node threads through `ThreadLaunchService`, reads their + * outcome from the domain event stream, and opens a pull request per branch end. + * + * Every graph change goes through one lock and is written whole, so a crash + * leaves either the old graph or the new one. Thread launches and git work run + * outside the lock against state already recorded as `running`/`delivering`, + * and are idempotent by command id so recovery can repeat them. + */ +export class TaskGraphService extends Context.Service< + TaskGraphService, + { + readonly create: (input: TaskGraphCreateInput) => Effect.Effect; + readonly get: (graphId: TaskGraphId) => Effect.Effect; + readonly listForThread: ( + threadId: ThreadId, + ) => Effect.Effect, TaskGraphError>; + /** Emits the thread's graphs on subscribe and again after each change to one of them. */ + readonly subscribeThread: ( + threadId: ThreadId, + ) => Stream.Stream; + readonly edit: ( + graphId: TaskGraphId, + edits: ReadonlyArray, + ) => Effect.Effect; + readonly run: (graphId: TaskGraphId) => Effect.Effect; + readonly cancel: (graphId: TaskGraphId) => Effect.Effect; + /** A node that ran on a peer finished there; recorded like a local node's end. */ + readonly completeRemoteNode: (input: { + readonly threadId: ThreadId; + readonly outcome: NodeOutcome; + }) => Effect.Effect; + } +>()("t3/taskGraph/TaskGraphService") {} + +export type NodeOutcome = + | { + readonly type: "succeeded"; + readonly summary: string | null; + readonly branch: string | null; + readonly worktreePath: string | null; + readonly pullRequestUrl?: string | null; + /** The node's PR step failed after its work was committed and pushed. */ + readonly pullRequestError?: string; + } + | { readonly type: "failed" | "cancelled"; readonly error: string }; + +/** Above this the host is too busy to take another node; matches client load balancing. */ +const MAX_CPU_UTILIZATION = 0.95; +const MIN_AVAILABLE_MEMORY_FRACTION = 0.05; + +function hostHasCapacity(snapshot: HostResourcesSnapshot): boolean { + if (snapshot.cpuUtilization !== null && snapshot.cpuUtilization >= MAX_CPU_UTILIZATION) { + return false; + } + return ( + snapshot.totalMemoryBytes <= 0 || + snapshot.availableMemoryBytes / snapshot.totalMemoryBytes > MIN_AVAILABLE_MEMORY_FRACTION + ); +} + +/** + * When a provider instance's usage limit resets, if one of its windows is used + * up right now; null when it has room or reports no limits. + */ +function usageLimitResetFor( + provider: Pick | undefined, + nowMs: number, +): string | null { + let latest: string | null = null; + for (const window of provider?.usageLimits?.windows ?? []) { + if (window.usedPercent < 100 || window.resetsAt === undefined) continue; + if (Date.parse(window.resetsAt) <= nowMs) continue; + if (latest === null || Date.parse(window.resetsAt) > Date.parse(latest)) + latest = window.resetsAt; + } + return latest; +} + +/** This machine's weight against its peers' (peers default to the same). */ +const LOCAL_WEIGHT = 50; + +const peerDelivery = (graph: TaskGraph, node: TaskGraphNode): TaskGraphPeers.PeerDelivery => + node.workspace === "root" + ? { action: "none", baseBranch: null } + : taskGraphNodeOpensPullRequest(graph.nodes, node) + ? { action: "commit_push_pr", baseBranch: taskGraphPullRequestBase(graph, node) } + : { action: "commit_push", baseBranch: null }; + +/** + * How a node's thread gets its workspace: a new worktree from its first + * dependency's branch (or the graph base), that dependency's worktree itself, + * or the project folder. Null when a continued worktree was never recorded. + */ +const nodeWorkspaceStrategy = ( + graph: TaskGraph, + node: TaskGraphNode, + options: { readonly baseFromOrigin: boolean }, +): ThreadLaunch.ThreadLaunchWorkspaceStrategy | null => { + const first = graph.nodes.find((candidate) => candidate.key === node.dependsOn[0]); + switch (node.workspace) { + case "root": + return { type: "root" }; + case "dependency": + return first?.worktreePath == null + ? null + : { + type: "existing_worktree", + worktreePath: first.worktreePath, + ...(first.branch === null ? {} : { branch: first.branch }), + }; + case "worktree": + return { + type: "worktree", + baseRef: first?.branch ?? graph.baseRef, + ...(options.baseFromOrigin ? { startFromOrigin: true } : {}), + }; + } +}; + +const decodeGraphJson = Schema.decodeUnknownEffect(Schema.fromJsonString(TaskGraph)); +const encodeGraphJson = Schema.encodeSync(Schema.fromJsonString(TaskGraph)); + +interface TaskGraphRow { + readonly graph_json: string; +} + +const graphError = (message: string, graphId?: TaskGraphId, cause?: unknown) => + new TaskGraphError({ + message, + ...(graphId === undefined ? {} : { graphId }), + ...(cause === undefined ? {} : { cause }), + }); + +function errorMessage(error: unknown): string { + if (Cause.isCause(error)) return Cause.pretty(error); + if (error instanceof Error) return error.message; + return String(error); +} + +const nowIso = DateTime.now.pipe(Effect.map((now) => DateTime.formatIso(now))); + +/** The message the proposing thread receives when its graph ends. */ +function taskGraphReport(graph: TaskGraph): string { + const lines = graph.nodes.map((node) => { + const pr = node.pullRequestResult; + const prText = + pr === null + ? "" + : pr.status === "opened" + ? ` — PR: ${pr.url ?? "opened"}` + : ` — PR failed: ${pr.error ?? "unknown error"}`; + const detail = node.status === "succeeded" ? "" : node.error === null ? "" : ` (${node.error})`; + return `- ${node.key} "${node.title}": ${node.status}${detail}${prText}`; + }); + return [`Task graph "${graph.title}" finished: ${graph.status}.`, ...lines].join("\n"); +} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const crypto = yield* Crypto.Crypto; + const launcher = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const git = yield* GitWorkflow.GitWorkflowService; + const hostResources = yield* HostResources.HostResources; + const providers = yield* ProviderRegistry.ProviderRegistry; + const settings = yield* ServerSettings.ServerSettingsService; + const scheduler = yield* Scheduler.Scheduler; + const environment = yield* ServerEnvironment.ServerEnvironment; + const peers = yield* TaskGraphPeers.TaskGraphPeers; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projects = yield* ProjectService.ProjectService; + const localEnvironmentId = yield* environment.getEnvironmentId; + + const lock = yield* Semaphore.make(1); + const locked = lock.withPermits(1); + const changes = yield* PubSub.unbounded(); + /** Node threads still running locally, so run events map back to their node. */ + const nodeByThread = new Map(); + /** Threads of nodes stopped by a usage limit, which continue there at the reset. */ + const waitingThreads = new Set(); + /** Threads of local nodes that failed or were cancelled, in case someone continues them. */ + const endedByThread = new Map(); + + const readGraph = (graphId: TaskGraphId) => + sql`SELECT graph_json FROM task_graphs WHERE graph_id = ${graphId}`.pipe( + Effect.flatMap((rows) => + rows[0] === undefined ? Effect.succeed(null) : decodeGraphJson(rows[0].graph_json), + ), + Effect.mapError((cause) => graphError("Could not read task graph.", graphId, cause)), + ); + + const requireGraph = (graphId: TaskGraphId) => + readGraph(graphId).pipe( + Effect.flatMap((graph) => + graph === null + ? Effect.fail(graphError("Task graph not found.", graphId)) + : Effect.succeed(graph), + ), + ); + + const readGraphs = (rows: ReadonlyArray) => + Effect.forEach(rows, (row) => decodeGraphJson(row.graph_json)).pipe( + Effect.mapError((cause) => graphError("Could not read task graphs.", undefined, cause)), + ); + + const runningGraphs = sql` + SELECT graph_json FROM task_graphs WHERE status = 'running' ORDER BY created_at + `.pipe( + Effect.mapError((cause) => graphError("Could not list task graphs.", undefined, cause)), + Effect.flatMap(readGraphs), + ); + + /** Graphs a node may still come back to life in: anything not cancelled as a whole. */ + const resumableGraphs = sql` + SELECT graph_json FROM task_graphs WHERE status IN ('running', 'failed') ORDER BY created_at + `.pipe( + Effect.mapError((cause) => graphError("Could not list task graphs.", undefined, cause)), + Effect.flatMap(readGraphs), + ); + + const indexGraph = (graph: TaskGraph) => { + for (const node of graph.nodes) { + if (node.threadId === null) continue; + const target = { graphId: graph.id, key: node.key }; + if (isActiveTaskGraphNodeStatus(node.status) || node.status === "waiting") { + nodeByThread.set(node.threadId, target); + } else { + nodeByThread.delete(node.threadId); + } + if (node.status === "waiting") waitingThreads.add(node.threadId); + else waitingThreads.delete(node.threadId); + if ( + (node.status === "failed" || node.status === "cancelled") && + graph.status !== "cancelled" && + node.assignedEnvironmentId === localEnvironmentId + ) { + endedByThread.set(node.threadId, target); + } else { + endedByThread.delete(node.threadId); + } + } + }; + + /** + * Someone continued a failed or cancelled node in its own thread. Follow + * that run instead of leaving the node, and everything after it, stuck. + */ + const resumeNode = (graphId: TaskGraphId, key: string) => + locked( + Effect.gen(function* () { + const graph = yield* requireGraph(graphId); + if (graph.status === "cancelled") return; + const nodes = resumeTaskGraphNode(graph.nodes, key); + if (nodes === null) return; + yield* writeGraph({ ...graph, status: "running", nodes }); + }), + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not resume task graph node", { + graphId, + key, + cause: Cause.pretty(cause), + }), + ), + ); + + /** Writes a graph after settling skips and status. Returns the stored graph. */ + const writeGraph = (graph: TaskGraph) => + Effect.gen(function* () { + const now = yield* nowIso; + const nodes = skipUnreachableTaskGraphNodes(graph.nodes, now); + const stored: TaskGraph = { + ...graph, + nodes, + status: deriveTaskGraphStatus(graph.status, nodes), + updatedAt: now, + }; + yield* sql` + INSERT INTO task_graphs (graph_id, project_id, thread_id, status, graph_json, created_at, updated_at) + VALUES (${stored.id}, ${stored.projectId}, ${stored.threadId}, ${stored.status}, + ${encodeGraphJson(stored)}, ${stored.createdAt}, ${stored.updatedAt}) + ON CONFLICT (graph_id) DO UPDATE SET + status = excluded.status, + graph_json = excluded.graph_json, + updated_at = excluded.updated_at + `.pipe(Effect.mapError((cause) => graphError("Could not save task graph.", graph.id, cause))); + indexGraph(stored); + yield* PubSub.publish(changes, stored.threadId); + return stored; + }); + + const updateNode = ( + graphId: TaskGraphId, + key: string, + update: (node: TaskGraphNode, graph: TaskGraph) => TaskGraphNode, + ) => + locked( + Effect.gen(function* () { + const graph = yield* requireGraph(graphId); + const before = graph.status; + const stored = yield* writeGraph({ + ...graph, + nodes: graph.nodes.map((node) => (node.key === key ? update(node, graph) : node)), + }); + return { stored, finished: before === "running" && stored.status !== "running" }; + }), + ).pipe(Effect.tap(({ stored, finished }) => (finished ? reportFinished(stored) : Effect.void))); + + /** Tells the proposing thread how its graph ended, queued behind any turn in progress. */ + const reportFinished = (graph: TaskGraph) => + threads + .sendToThread({ + projectId: graph.projectId, + commandId: CommandId.make(`task-graph-report:${graph.id}:${graph.updatedAt}`), + threadId: graph.threadId, + messageId: MessageId.make(`task-graph-report:${graph.id}:${graph.updatedAt}`), + text: taskGraphReport(graph), + attachments: [], + mode: "queue", + createdBy: "system", + creationSource: "server", + }) + .pipe( + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logWarning("Could not report finished task graph", { + graphId: graph.id, + cause: Cause.pretty(cause), + }), + ), + ); + + /** A node held by a usage limit is running again on its thread. */ + const wakeNode = (graphId: TaskGraphId, key: string) => + updateNode(graphId, key, (node) => + node.status === "waiting" && node.threadId !== null + ? { ...node, status: "running", waitUntil: null, waitReason: null } + : node, + ).pipe( + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logWarning("Could not wake task graph node", { + graphId, + key, + cause: Cause.pretty(cause), + }), + ), + ); + + // --- Finishing nodes ------------------------------------------------------- + + /** Commits the node's work so dependents can branch from it, and opens its PR. */ + const deliver = (graph: TaskGraph, node: TaskGraphNode, worktreePath: string) => + Effect.gen(function* () { + const opensPullRequest = taskGraphNodeOpensPullRequest(graph.nodes, node); + // With peers paired, a dependent may run elsewhere and needs the branch on origin. + const push = + (yield* peers.hasPeers) || taskGraphNodeSharesPullRequestBranch(graph.nodes, node); + const baseBranch = opensPullRequest ? taskGraphPullRequestBase(graph, node) : null; + let failedPhase: string | null = null; + const delivered = yield* git + .runStackedAction( + { + actionId: `task-graph:${graph.id}:${node.key}:${node.threadId}`, + cwd: worktreePath, + action: opensPullRequest ? "commit_push_pr" : push ? "commit_push" : "commit", + ...(baseBranch === null ? {} : { baseBranch }), + ...(node.threadId === null ? {} : { threadId: node.threadId }), + projectId: graph.projectId, + }, + { + progressReporter: { + publish: (event) => + Effect.sync(() => { + if (event.kind === "action_failed") failedPhase = event.phase; + }), + }, + }, + ) + .pipe( + // Only a failed PR step leaves the work committed and pushed for dependents; + // a commit or push failure fails the node. + Effect.map((result) => ({ result, prError: null })), + Effect.catch((error) => + opensPullRequest && failedPhase === "pr" + ? Effect.succeed({ result: null, prError: error.message }) + : Effect.fail(error), + ), + ); + if (delivered.result === null) { + return { status: "failed", url: null, error: delivered.prError } as const; + } + const result = delivered.result; + if (opensPullRequest && node.threadId !== null) { + yield* linkCreatedPullRequest({ + threadId: node.threadId, + result, + commandId: Effect.succeed(CommandId.make(`task-graph-pr-link:${node.threadId}`)), + }).pipe( + Effect.provideService(Orchestrator.OrchestratorV2, orchestrator), + Effect.provideService(ProjectService.ProjectService, projects), + ); + } + return opensPullRequest + ? ({ status: "opened", url: result.pr.url ?? null, error: null } as const) + : null; + }); + + /** A succeeded node's thread has nothing left to do; settle it so it leaves the active list. */ + const settleNodeThread = (threadId: ThreadId) => + threads + .settleThread({ + threadId, + commandId: CommandId.make(`task-graph-settle:${threadId}`), + byOwnAgent: false, + }) + .pipe( + Effect.asVoid, + Effect.catchCause((cause) => + Effect.logWarning("Could not settle a succeeded task graph node's thread", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ); + + const finishLocalNode = (graphId: TaskGraphId, key: string, threadId: ThreadId) => + Effect.gen(function* () { + const projection = yield* threads.getThreadProjection(threadId); + // A follow-up the user sent to the node thread keeps the node running. + if (ThreadManagement.latestActiveRun(projection) !== undefined) return; + const run = ThreadManagement.latestRun(projection); + nodeByThread.delete(threadId); + const limited = usageLimitBlockedRun(projection.runs, projection.turnItems, null); + const resetAt = + limited === null + ? null + : (latestRootProviderFailure(limited, projection.turnItems)?.resetAt ?? null); + if (limited !== null && resetAt !== null && limited.id === run?.id) { + // Hold the node and let usage-limit recovery continue its thread at the reset. + yield* updateNode(graphId, key, (node) => + node.status === "running" + ? { ...node, status: "waiting", waitUntil: resetAt, waitReason: "usage_limit" } + : node, + ); + yield* threads + .dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`task-graph-limit-arm:${threadId}:${limited.id}`), + threadId, + limitRecovery: { runId: limited.id, resetAt, autoResume: true }, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not arm usage limit recovery for a task graph node", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ); + return; + } + if (run === undefined || run.status !== "completed") { + const completedAt = yield* nowIso; + const status = + run?.status === "cancelled" || run?.status === "interrupted" ? "cancelled" : "failed"; + yield* updateNode(graphId, key, (node, _graph) => + node.status === "running" + ? { + ...node, + status, + error: + run === undefined + ? "The node's thread has no run." + : `The node's run ${run.status}.`, + completedAt, + } + : node, + ).pipe(Effect.tap(() => advance)); + return; + } + const lastReply = projection.messages.findLast( + (message) => message.role === "assistant" && message.text.trim().length > 0, + ); + // Completions run concurrently, so only the call that moves the node out of + // `running` under the lock delivers it; any overlapping call stops here. + let claimed = false; + const { stored } = yield* updateNode(graphId, key, (node) => { + if (node.status !== "running") return node; + claimed = true; + return { + ...node, + status: "delivering", + branch: projection.thread.branch, + worktreePath: projection.thread.worktreePath, + summary: lastReply === undefined ? null : taskGraphNodeSummary(lastReply.text), + }; + }); + const node = stored.nodes.find((candidate) => candidate.key === key); + if (!claimed || node === undefined || node.status !== "delivering") return; + const worktreePath = projection.thread.worktreePath; + // Work in the project folder is never committed. + const delivered = + worktreePath === null || node.workspace === "root" + ? Exit.succeed(null) + : yield* Effect.exit(deliver(stored, node, worktreePath)); + const completedAt = yield* nowIso; + const finished = yield* updateNode(graphId, key, (current) => + current.status !== "delivering" + ? current + : Exit.isSuccess(delivered) + ? // A failed PR alone still succeeds: the work is committed and pushed for dependents. + { ...current, status: "succeeded", pullRequestResult: delivered.value, completedAt } + : { + ...current, + status: "failed", + error: `Could not commit the node's work: ${errorMessage(delivered.cause)}`, + completedAt, + }, + ); + if ( + finished.stored.nodes.find((candidate) => candidate.key === key)?.status === "succeeded" + ) { + yield* settleNodeThread(threadId); + } + yield* advance; + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not finish task graph node", { + graphId, + key, + cause: Cause.pretty(cause), + }), + ), + ); + + const completeRemoteNode: TaskGraphService["Service"]["completeRemoteNode"] = ({ + threadId, + outcome, + }) => + Effect.gen(function* () { + const target = nodeByThread.get(threadId); + if (target === undefined) return; + nodeByThread.delete(threadId); + const completedAt = yield* nowIso; + yield* updateNode(target.graphId, target.key, (node) => { + if (!isActiveTaskGraphNodeStatus(node.status)) return node; + if (outcome.type !== "succeeded") { + return { ...node, status: outcome.type, error: outcome.error, completedAt }; + } + return { + ...node, + status: "succeeded", + summary: outcome.summary === null ? null : taskGraphNodeSummary(outcome.summary), + branch: outcome.branch, + worktreePath: outcome.worktreePath, + pullRequestResult: + outcome.pullRequestError !== undefined + ? { status: "failed", url: null, error: outcome.pullRequestError } + : outcome.pullRequestUrl === undefined + ? null + : { status: "opened", url: outcome.pullRequestUrl, error: null }, + completedAt, + }; + }); + yield* advance; + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not record remote task graph node", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ); + + // --- Starting nodes -------------------------------------------------------- + + /** + * Where a ready node runs: its pinned machine, or whichever of this machine + * and its peers has the most free capacity, scored like client load balancing. + * Null when nowhere has room; the node waits for the next pass. + */ + const placeNode = ( + graph: TaskGraph, + node: TaskGraphNode, + local: { readonly resources: HostResourcesSnapshot; readonly hasRoom: boolean }, + maxNodes: number, + peerCandidates: ReadonlyArray, + pending: ReadonlyMap, + ): Effect.Effect => + Effect.gen(function* () { + // Continuing a worktree means running on the machine that has it. + const continued = + node.workspace === "dependency" + ? (graph.nodes.find((candidate) => candidate.key === node.dependsOn[0]) + ?.assignedEnvironmentId ?? null) + : null; + const pinned = continued ?? node.environmentId; + if (pinned === localEnvironmentId) { + return local.hasRoom ? localEnvironmentId : null; + } + // Peers count a node once it starts there, after this pass; this pass's placements count too. + const remote = peerCandidates.filter( + (candidate) => candidate.running + (pending.get(candidate.environmentId) ?? 0) < maxNodes, + ); + if (pinned !== null) { + return remote.some((candidate) => candidate.environmentId === pinned) ? pinned : null; + } + if (remote.length === 0) return local.hasRoom ? localEnvironmentId : null; + const now = yield* Clock.currentTimeMillis; + const chosen = chooseLoadBalancedEnvironment( + [ + ...(local.hasRoom + ? [ + { + environmentId: localEnvironmentId, + resources: local.resources, + receivedAt: now, + weight: LOCAL_WEIGHT, + }, + ] + : []), + ...remote, + ], + now, + ); + // The scorer skips a machine without a CPU sample yet; this one can still take work. + if (chosen === null) return local.hasRoom ? localEnvironmentId : null; + return EnvironmentId.make(chosen); + }); + + const launchNode = (graph: TaskGraph, node: TaskGraphNode) => + Effect.gen(function* () { + const threadId = node.threadId!; + const parent = yield* threads.getThreadProjection(graph.threadId); + const firstDependency = graph.nodes.find((candidate) => candidate.key === node.dependsOn[0]); + const prompt = buildTaskGraphNodePrompt(graph, node); + const modelSelection = + node.modelSelection ?? graph.modelSelection ?? parent.thread.modelSelection; + const remote = node.assignedEnvironmentId !== localEnvironmentId; + // A dependency that ran on another machine pushed its branch; it is only on origin here. + const workspaceStrategy = nodeWorkspaceStrategy(graph, node, { + baseFromOrigin: + remote || + (firstDependency !== undefined && + firstDependency.assignedEnvironmentId !== localEnvironmentId), + }); + if (workspaceStrategy === null) { + return yield* graphError( + `'${node.dependsOn[0]}' left no worktree to continue in.`, + graph.id, + ); + } + if (remote) { + yield* peers.startNode({ + environmentId: node.assignedEnvironmentId!, + graph, + node, + threadId, + workspaceStrategy, + prompt, + modelSelection, + runtimeMode: parent.thread.runtimeMode, + delivery: peerDelivery(graph, node), + }); + return; + } + yield* launcher.launch({ + commandId: CommandId.make(`task-graph-launch:${threadId}`), + threadId, + projectId: graph.projectId, + title: node.title, + modelSelection, + runtimeMode: parent.thread.runtimeMode, + interactionMode: "default", + workspaceStrategy, + initialMessage: { + messageId: MessageId.make(`task-graph-message:${threadId}`), + senderThreadId: graph.threadId, + text: prompt, + attachments: [], + }, + createdBy: "agent", + creationSource: "server", + }); + }).pipe( + Effect.catchCause((cause) => + Effect.gen(function* () { + const completedAt = yield* nowIso; + nodeByThread.delete(node.threadId!); + yield* updateNode(graph.id, node.key, (current) => + current.status === "running" + ? { + ...current, + status: "failed", + error: `Could not start the node: ${errorMessage(cause)}`, + completedAt, + } + : current, + ); + }), + ), + ); + + /** + * Starts every node that is ready and fits: under the environment's + * concurrency limit, and on a machine that is not overloaded. Nodes that do + * not fit wait for the next scheduler tick or the next node to finish. + */ + const advance: Effect.Effect = Effect.gen(function* () { + const { taskGraphMaxConcurrentNodes } = yield* settings.getSettings; + // Asking peers for their load is a network call per peer, so it happens once per pass and + // outside the lock that edits, cancels and completions wait on. + const projectIds = [...new Set((yield* runningGraphs).map((graph) => graph.projectId))]; + const peerCandidates = new Map( + yield* Effect.forEach( + projectIds, + (projectId) => + peers.candidates({ projectId }).pipe(Effect.map((found) => [projectId, found] as const)), + { concurrency: "unbounded" }, + ), + ); + const toLaunch = yield* locked( + Effect.gen(function* () { + const graphs = yield* runningGraphs; + let active = graphs + .flatMap((graph) => graph.nodes) + .filter( + (node) => + isActiveTaskGraphNodeStatus(node.status) && + node.assignedEnvironmentId === localEnvironmentId, + ).length; + const host = yield* hostResources.read; + const hostFree = hostHasCapacity(host); + const started: Array<{ graph: TaskGraph; node: TaskGraphNode }> = []; + const now = yield* nowIso; + const nowMs = Date.parse(now); + const localProviders = yield* providers.getProviders; + const placedOnPeers = new Map(); + for (const graph of graphs) { + const settled = skipUnreachableTaskGraphNodes(graph.nodes, now); + const ready = readyTaskGraphNodes(settled, now); + const assigned = new Map(); + for (const node of ready) { + if (node.startAt !== null && Date.parse(node.startAt) > nowMs) { + if (node.status !== "waiting") { + assigned.set(node.key, { + ...node, + status: "waiting", + waitUntil: node.startAt, + waitReason: "scheduled", + }); + } + continue; + } + const environmentId = yield* placeNode( + graph, + node, + { resources: host, hasRoom: hostFree && active < taskGraphMaxConcurrentNodes }, + taskGraphMaxConcurrentNodes, + peerCandidates.get(graph.projectId) ?? [], + placedOnPeers, + ); + if (environmentId === null) continue; + if (environmentId !== localEnvironmentId) { + placedOnPeers.set(environmentId, (placedOnPeers.get(environmentId) ?? 0) + 1); + } + if (environmentId === localEnvironmentId) { + // Starting into a used-up limit would only fail; wait for the reset instead. + const instanceId = (node.modelSelection ?? graph.modelSelection)?.instanceId; + const resetAt = usageLimitResetFor( + localProviders.find((provider) => provider.instanceId === instanceId), + nowMs, + ); + if (resetAt !== null) { + if (node.waitUntil !== resetAt) { + assigned.set(node.key, { + ...node, + status: "waiting", + waitUntil: resetAt, + waitReason: "usage_limit", + }); + } + continue; + } + active += 1; + } + assigned.set(node.key, { + ...node, + status: "running", + waitUntil: null, + waitReason: null, + assignedEnvironmentId: environmentId, + threadId: ThreadId.make(`task-graph-node:${yield* crypto.randomUUIDv4}`), + startedAt: yield* nowIso, + }); + } + // Skips can settle a graph with nothing to start, but an unchanged graph + // is not rewritten: every write is pushed to each client watching it. + const nodes = settled.map((node) => assigned.get(node.key) ?? node); + if (nodes.every((node, index) => node === graph.nodes[index])) continue; + const stored = yield* writeGraph({ ...graph, nodes }); + if (stored.status !== "running") yield* reportFinished(stored).pipe(Effect.forkDetach); + // Nodes set waiting above have no thread yet; they launch on a later pass. + for (const node of assigned.values()) { + if (node.status === "running") started.push({ graph: stored, node }); + } + } + return started; + }), + ); + yield* Effect.forEach(toLaunch, ({ graph, node }) => launchNode(graph, node), { + concurrency: "unbounded", + discard: true, + }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not advance task graphs", { cause: Cause.pretty(cause) }), + ), + ); + + const stopNodes = (graph: TaskGraph, nodes: ReadonlyArray) => + Effect.forEach( + nodes, + (node) => { + if (node.threadId === null) return Effect.void; + nodeByThread.delete(node.threadId); + if ( + node.assignedEnvironmentId !== localEnvironmentId && + node.assignedEnvironmentId !== null + ) { + return peers.interruptNode({ + environmentId: node.assignedEnvironmentId, + threadId: node.threadId, + }); + } + const disarm = + node.status === "waiting" + ? threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`task-graph-limit-disarm:${node.threadId}`), + threadId: node.threadId, + limitRecovery: null, + }) + : Effect.void; + return Effect.andThen( + disarm, + threads + .interruptThread({ + projectId: graph.projectId, + commandId: CommandId.make(`task-graph-stop:${node.threadId}`), + threadId: node.threadId, + reason: "The task graph branch was cancelled.", + }) + .pipe(Effect.asVoid), + ); + }, + { discard: true }, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not stop cancelled task graph nodes", { + graphId: graph.id, + cause: Cause.pretty(cause), + }), + ), + ); + + // --- Public API ------------------------------------------------------------- + + const create: TaskGraphService["Service"]["create"] = (input) => + Effect.gen(function* () { + const parent = yield* threads + .getThreadProjection(input.threadId) + .pipe( + Effect.mapError((cause) => graphError("Could not read the thread.", undefined, cause)), + ); + const nodes = input.nodes.map(newTaskGraphNode); + const invalid = validateTaskGraphNodes(nodes); + if (invalid !== null) return yield* graphError(invalid); + const { taskGraphAutoRun } = yield* settings.getSettings.pipe( + Effect.mapError((cause) => graphError("Could not read settings.", undefined, cause)), + ); + const now = yield* nowIso; + const uuid = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => graphError("Could not create task graph id.", undefined, cause)), + ); + const graph = yield* locked( + writeGraph({ + id: TaskGraphId.make(`task-graph:${uuid}`), + projectId: parent.thread.projectId, + threadId: parent.thread.id, + title: input.title, + baseRef: input.baseRef ?? parent.thread.branch ?? "HEAD", + modelSelection: parent.thread.modelSelection, + status: (input.run ?? taskGraphAutoRun) ? "running" : "draft", + nodes, + createdAt: now, + updatedAt: now, + }), + ); + if (graph.status === "running") yield* advance; + return yield* requireGraph(graph.id); + }); + + const edit: TaskGraphService["Service"]["edit"] = (graphId, edits) => + Effect.gen(function* () { + const { graph, stopped, finished } = yield* locked( + Effect.gen(function* () { + const current = yield* requireGraph(graphId); + if (current.status === "cancelled") { + return yield* graphError("This task graph was cancelled.", graphId); + } + const result = applyTaskGraphEdits(current.nodes, edits, yield* nowIso); + if (!result.ok) return yield* graphError(result.error, graphId); + const stopped = current.nodes.filter((node) => { + const next = result.nodes.find((candidate) => candidate.key === node.key); + return ( + (isActiveTaskGraphNodeStatus(node.status) || + (node.status === "waiting" && node.threadId !== null)) && + next?.status === "cancelled" + ); + }); + // Retrying a node in a finished graph puts the graph back to work. + const reopened = result.nodes.some((node) => node.status === "pending"); + const status = + current.status === "draft" ? "draft" : reopened ? "running" : current.status; + const graph = yield* writeGraph({ ...current, status, nodes: result.nodes }); + return { + graph, + stopped, + finished: current.status === "running" && graph.status !== "running", + }; + }), + ); + yield* stopNodes(graph, stopped); + if (finished) yield* reportFinished(graph); + if (graph.status === "running") yield* advance; + return yield* requireGraph(graphId); + }); + + const run: TaskGraphService["Service"]["run"] = (graphId) => + Effect.gen(function* () { + yield* locked( + Effect.gen(function* () { + const graph = yield* requireGraph(graphId); + if (graph.status !== "draft") { + return yield* graphError("Only a draft task graph can be run.", graphId); + } + yield* writeGraph({ ...graph, status: "running" }); + }), + ); + yield* advance; + return yield* requireGraph(graphId); + }); + + const cancel: TaskGraphService["Service"]["cancel"] = (graphId) => + Effect.gen(function* () { + const { graph, stopped } = yield* locked( + Effect.gen(function* () { + const current = yield* requireGraph(graphId); + const completedAt = yield* nowIso; + const nodes = current.nodes.map((node) => + isTerminalTaskGraphNodeStatus(node.status) + ? node + : { ...node, status: "cancelled" as const, error: "Cancelled.", completedAt }, + ); + const graph = yield* writeGraph({ ...current, status: "cancelled", nodes }); + return { + graph, + stopped: current.nodes.filter( + (node) => + isActiveTaskGraphNodeStatus(node.status) || + (node.status === "waiting" && node.threadId !== null), + ), + }; + }), + ); + yield* stopNodes(graph, stopped); + return graph; + }); + + const listForThread: TaskGraphService["Service"]["listForThread"] = (threadId) => + sql` + SELECT graph_json FROM task_graphs WHERE thread_id = ${threadId} ORDER BY created_at + `.pipe( + Effect.mapError((cause) => graphError("Could not list task graphs.", undefined, cause)), + Effect.flatMap(readGraphs), + ); + + const subscribeThread: TaskGraphService["Service"]["subscribeThread"] = (threadId) => + Stream.unwrap( + Effect.gen(function* () { + // Subscribe before the snapshot so a change between the two is not lost. + const subscription = yield* PubSub.subscribe(changes); + return Stream.concat( + Stream.fromEffect(listForThread(threadId)), + Stream.fromSubscription(subscription).pipe( + Stream.filter((changed) => changed === threadId), + Stream.mapEffect(() => listForThread(threadId)), + ), + ).pipe(Stream.map((graphs) => ({ graphs }))); + }), + ); + + // --- Startup ---------------------------------------------------------------- + + // Rebuild the thread index, then catch up on node runs that ended while the + // server was down. Launches are idempotent by command id, so a node that was + // marked running but never launched is launched again. + yield* Effect.gen(function* () { + for (const graph of yield* resumableGraphs) indexGraph(graph); + // A node continued in its thread while the server was down: follow it, and + // record it if that run already finished after the node had failed. + yield* Effect.forEach( + [...endedByThread], + ([threadId, { graphId, key }]) => + Effect.gen(function* () { + const projection = yield* threads.getThreadProjection(threadId); + const graph = yield* requireGraph(graphId); + const endedAt = graph.nodes.find((node) => node.key === key)?.completedAt ?? null; + const run = ThreadManagement.latestRun(projection); + const ranAgain = + ThreadManagement.latestActiveRun(projection) !== undefined || + (run?.completedAt != null && + endedAt !== null && + DateTime.toEpochMillis(run.completedAt) > + DateTime.toEpochMillis(DateTime.makeUnsafe(endedAt))); + if (!ranAgain) return; + yield* resumeNode(graphId, key); + yield* finishLocalNode(graphId, key, threadId); + }).pipe(Effect.ignoreCause), + { discard: true }, + ); + const graphs = yield* runningGraphs; + // A node held by a usage limit whose thread was continued while the server was down. + yield* Effect.forEach( + graphs.flatMap((graph) => + graph.nodes + .filter((node) => node.status === "waiting" && node.threadId !== null) + .map((node) => ({ graph, node })), + ), + ({ graph, node }) => + Effect.gen(function* () { + const projection = yield* threads.getThreadProjection(node.threadId!); + if (ThreadManagement.latestRun(projection)?.status === "failed") return; + yield* wakeNode(graph.id, node.key); + yield* finishLocalNode(graph.id, node.key, node.threadId!); + }).pipe(Effect.ignoreCause), + { discard: true }, + ); + yield* Effect.forEach( + graphs.flatMap((graph) => + graph.nodes + .filter( + (node) => + node.status === "running" && node.assignedEnvironmentId === localEnvironmentId, + ) + .map((node) => ({ graph, node })), + ), + ({ graph, node }) => + threads.getThreadProjection(node.threadId!).pipe( + Effect.matchEffect({ + onFailure: () => launchNode(graph, node), + onSuccess: () => finishLocalNode(graph.id, node.key, node.threadId!), + }), + ), + { discard: true }, + ); + yield* Effect.forEach( + graphs.flatMap((graph) => + graph.nodes + .filter( + (node) => + isActiveTaskGraphNodeStatus(node.status) && + node.assignedEnvironmentId !== null && + node.assignedEnvironmentId !== localEnvironmentId, + ) + .map((node) => ({ graph, node })), + ), + ({ graph, node }) => + peers.track({ + environmentId: node.assignedEnvironmentId!, + threadId: node.threadId!, + delivery: peerDelivery(graph, node), + }), + { discard: true }, + ); + // A node mid-delivery when the server stopped redelivers; the git action is idempotent. + yield* Effect.forEach( + graphs.flatMap((graph) => + graph.nodes + .filter( + (node) => + node.status === "delivering" && node.assignedEnvironmentId === localEnvironmentId, + ) + .map((node) => ({ graph, node })), + ), + ({ graph, node }) => + updateNode(graph.id, node.key, (current) => ({ ...current, status: "running" })).pipe( + Effect.andThen(finishLocalNode(graph.id, node.key, node.threadId!)), + ), + { discard: true }, + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not recover task graphs", { cause: Cause.pretty(cause) }), + ), + ); + + yield* forkParked( + Stream.runForEach(threads.streamDomainEvents, (event) => { + if (event.type !== "run.updated") return Effect.void; + if (!ThreadManagement.isTerminalRunStatus(event.payload.status)) { + const waiting = waitingThreads.has(event.threadId) + ? nodeByThread.get(event.threadId) + : undefined; + if (waiting !== undefined) return wakeNode(waiting.graphId, waiting.key); + // Only a new run passes through an active status, so a repeated update of + // the run that failed never resumes the node. + const ended = endedByThread.get(event.threadId); + return ended === undefined ? Effect.void : resumeNode(ended.graphId, ended.key); + } + const target = nodeByThread.get(event.threadId); + // Delivery commits, pushes and writes a PR, so finish each node in its own fiber + // rather than hold up every other graph's events. Status guards make a repeat a no-op. + return target === undefined + ? Effect.void + : finishLocalNode(target.graphId, target.key, event.threadId).pipe(Effect.forkDetach); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Task graph event stream failed", { cause: Cause.pretty(cause) }), + ), + ), + ); + yield* forkParked(Stream.runForEach(peers.completions, completeRemoteNode)); + + // Retries nodes held back by load or the concurrency limit, and picks up peer capacity. + yield* scheduler.register("task-graphs", advance); + + return TaskGraphService.of({ + create, + get: requireGraph, + listForThread, + subscribeThread, + edit, + run, + cancel, + completeRemoteNode, + }); +}); + +export const layer = Layer.effect(TaskGraphService, make); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 80921ec4b5a2..cff52cd272a3 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -123,6 +123,8 @@ import * as ThreadLaunchService from "./orchestration-v2/ThreadLaunchService.ts" import * as ThreadMessageIntake from "./orchestration-v2/ThreadMessageIntake.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ScheduledTasks from "./scheduledTasks/ScheduledTaskService.ts"; +import * as TaskGraphPeers from "./taskGraph/TaskGraphPeers.ts"; +import * as TaskGraphs from "./taskGraph/TaskGraphService.ts"; import * as SecretRequests from "./secrets/SecretRequests.ts"; import { archivedShellStreamItemFromThreadShell, @@ -1224,6 +1226,8 @@ const layerWsRpc = ( const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService; const providerSessionManager = yield* ProviderSessionManager.ProviderSessionManagerV2; const scheduledTasks = yield* ScheduledTasks.ScheduledTaskService; + const taskGraphs = yield* TaskGraphs.TaskGraphService; + const taskGraphPeers = yield* TaskGraphPeers.TaskGraphPeers; const secretRequests = yield* SecretRequests.SecretRequests; const pullRequests = yield* PullRequestService.PullRequestService; const reviewGuides = yield* PullRequestReviewGuideService.PullRequestReviewGuideService; @@ -2048,6 +2052,21 @@ const layerWsRpc = ( [WS_METHODS.scheduledTasksSubscribe]: (_input) => scheduledTasks.subscribeList().pipe(Stream.map(withVisibleWebhookUrls)), [WS_METHODS.scheduledTasksUpsert]: (input) => scheduledTasks.upsert(input), + [WS_METHODS.taskGraphsSubscribe]: (input) => taskGraphs.subscribeThread(input.threadId), + [WS_METHODS.taskGraphsCreate]: (input) => + taskGraphs.create(input).pipe(Effect.map((graph) => ({ graph }))), + [WS_METHODS.taskGraphsEdit]: (input) => + taskGraphs.edit(input.graphId, input.edits).pipe(Effect.map((graph) => ({ graph }))), + [WS_METHODS.taskGraphsRun]: (input) => + taskGraphs.run(input.graphId).pipe(Effect.map((graph) => ({ graph }))), + [WS_METHODS.taskGraphsCancel]: (input) => + taskGraphs.cancel(input.graphId).pipe(Effect.map((graph) => ({ graph }))), + [WS_METHODS.taskGraphPeersSubscribe]: (_input) => taskGraphPeers.subscribe, + [WS_METHODS.taskGraphPeersAdd]: (input) => taskGraphPeers.add(input), + [WS_METHODS.taskGraphPeersRemove]: (input) => taskGraphPeers.remove(input.environmentId), + [WS_METHODS.taskGraphPeersIssueGrant]: (input) => taskGraphPeers.issueGrant(input.label), + [WS_METHODS.taskGraphPeersSetWeight]: (input) => + taskGraphPeers.setWeight(input.environmentId, input.weight), [WS_METHODS.scheduledTasksSetEnabled]: (input) => Effect.annotateCurrentSpan({ "scheduled_task.id": input.id }).pipe( Effect.andThen(scheduledTasks.setEnabled(input)), diff --git a/apps/web/package.json b/apps/web/package.json index 4d10f89743b6..5b8168776074 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -15,6 +15,7 @@ "@base-ui/react": "^1.4.1", "@clerk/electron": "catalog:", "@clerk/react": "catalog:", + "@dagrejs/dagre": "^3.1.1", "@daypicker/react": "^10.0.1", "@dnd-kit/core": "^6.3.1", "@dnd-kit/modifiers": "^9.0.0", @@ -50,6 +51,7 @@ "@tiptap/pm": "^3.31.3", "@tiptap/react": "^3.31.3", "@tiptap/starter-kit": "^3.31.3", + "@xyflow/react": "^12.12.0", "class-variance-authority": "^0.7.1", "culori": "^4.0.2", "dompurify": "^3.4.16", diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index 1e79964caa0a..5ba4dda5eb44 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -1104,6 +1104,19 @@ export async function waitForStartedServerThread( }); } +/** Resolves true once a thread created by this client shows up in its shell list. */ +export async function waitForServerThreadShell( + threadRef: ScopedThreadRef, + timeoutMs = 5_000, +): Promise { + return waitForAtomValue({ + registry: appAtomRegistry, + atom: environmentThreadShells.threadShellAtom(threadRef), + predicate: (thread) => thread != null, + timeoutMs, + }); +} + /** * Runs `revert` (the rollback command `requestId`) and resolves once the * message's run is rolled back. Rejects with the server's reason as soon as diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 84f5b476a133..6e878c711c6c 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -83,6 +83,7 @@ import { resolveEnvironmentMachineKind, RuntimeMode, TerminalOpenInput, + type TaskGraphNode, type WorktreeSetupSnapshot, } from "@t3tools/contracts"; import { type EnvironmentConnectionPresentation } from "@t3tools/client-runtime/connection"; @@ -487,6 +488,8 @@ import { type QueuedRunsControlHandle, type EditQueuedRunRequest, } from "./chat/QueuedRunsControl"; +import { DraftTaskGraphDialog, TaskGraphCards } from "./chat/TaskGraphCard"; +import { useNewTaskGraph } from "./chat/useTaskGraphCommands"; import { useLinkedThreadPullRequest } from "./ThreadStatusIndicators"; import type { ComposerBannerStackItem } from "./chat/ComposerBannerStack"; import { ThreadStatusLine } from "./chat/ThreadStatusLine"; @@ -556,6 +559,7 @@ import { startNewThreadForProject, codexArtifactTemplatePromptToAppend, waitForStartedServerThread, + waitForServerThreadShell, shouldRefocusComposerOnWindowFocus, } from "./ChatView.logic"; import { useLocalStorage } from "~/hooks/useLocalStorage"; @@ -10735,6 +10739,91 @@ export default function ChatView(props: ChatViewProps) { composerRef, ]); + const newTaskGraph = useNewTaskGraph(environmentId); + // On a new chat the graph is drafted in the client; its thread and the + // graph are only created when the user runs it. + const [draftTaskGraphOpen, setDraftTaskGraphOpen] = useState(false); + // The composer's model when the draft opened: shown as the graph default and used on Run. + const [draftTaskGraphModel, setDraftTaskGraphModel] = useState(null); + const onNewTaskGraph = useCallback(async () => { + if (!activeThread) return; + if (!isServerThread) { + setDraftTaskGraphModel(composerRef.current?.getSendContext()?.selectedModelSelection ?? null); + setDraftTaskGraphOpen(true); + return; + } + const error = await newTaskGraph.start(activeThread.id); + if (error !== null) { + toastManager.add({ + type: "error", + title: "Could not start a task graph", + description: error, + }); + } + }, [activeThread, composerRef, isServerThread, newTaskGraph]); + const onRunDraftTaskGraph = useCallback( + async (nodes: ReadonlyArray): Promise => { + const modelSelection = draftTaskGraphModel; + if (!activeThread || !activeProject || !modelSelection) { + return "Pick a project and model before running the graph."; + } + const nextThreadId = newThreadId(); + const createResult = await createThread({ + environmentId, + input: { + threadId: nextThreadId, + projectId: activeProject.id, + title: "New task graph", + modelSelection, + runtimeMode, + interactionMode: "default", + branch: activeThreadBranch, + worktreePath: activeThread.worktreePath, + createdAt: new Date().toISOString(), + }, + }); + if (createResult._tag === "Failure") { + const failure = squashAtomCommandFailure(createResult); + return failure instanceof Error ? failure.message : "Could not create the thread."; + } + const created = await newTaskGraph.createAndRun(nextThreadId, nodes); + if (created.outcome === "failed") { + await deleteThread({ environmentId, input: { threadId: nextThreadId } }); + return created.error; + } + // An interrupted request may have started the graph: keep its thread and open it + // rather than risk running the same tasks twice from the draft. + if (created.outcome === "uncertain") { + toastManager.add({ + type: "warning", + title: "The task graph request was interrupted", + description: "Check the new chat to see whether the graph started.", + }); + } + // The graph is running now, so its thread stays even if the shell is slow to arrive; + // waiting only keeps the route from opening before the thread is known. + await waitForServerThreadShell(scopeThreadRef(environmentId, nextThreadId)); + setDraftTaskGraphOpen(false); + await navigate({ + to: "/$environmentId/$threadId", + params: { environmentId, threadId: nextThreadId }, + }); + return null; + }, + [ + activeProject, + activeThread, + activeThreadBranch, + createThread, + deleteThread, + draftTaskGraphModel, + environmentId, + navigate, + newTaskGraph, + runtimeMode, + ], + ); + const getModelDisabledReason = useCallback( (instanceId: ProviderInstanceId, model: string): string | null => { if (!activeThread) { @@ -11665,6 +11754,11 @@ export default function ChatView(props: ChatViewProps) { promptHistoryMessages={timelineMessages} isServerThread={isServerThread} isLocalDraftThread={isLocalDraftThread} + onNewTaskGraph={ + newTaskGraph.canCreate && canOperateThread + ? () => void onNewTaskGraph() + : null + } forceExpandedOnMobile={ forceExpandedMobileComposer && isDraftHeroState } @@ -11701,25 +11795,31 @@ export default function ChatView(props: ChatViewProps) { isPreparingWorktree={isPreparingWorktree && !sendQueuesBehindSetup} queuedRunsControl={ isServerThread && activeThread ? ( - + <> + + + ) : null } bannerItems={composerBannerItems} @@ -12137,6 +12237,18 @@ export default function ChatView(props: ChatViewProps) { + {!isServerThread && activeThread && activeProject ? ( + + ) : null} {expandedImage && ( ; isServerThread: boolean; isLocalDraftThread: boolean; + /** Starts a task graph on this thread, creating the thread first on a new chat. */ + onNewTaskGraph: (() => void) | null; forceExpandedOnMobile: boolean; projectSelectionRequired: boolean; @@ -1646,7 +1649,7 @@ export interface ChatComposerProps { onPageScrollKeyUp: (key: string) => void; onPageScrollRelease: () => void; - // Queued runs strip rendered above the composer (v2 queue/steer). + // Attachments rendered above the composer: task graphs and the queued runs strip. queuedRunsControl?: ReactNode; // Queued-message edit mode: attachments already stored on the message being // edited. Rendered in the attachment strip with a remove control; removal is @@ -1726,6 +1729,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) promptHistoryMessages, isServerThread: _isServerThread, isLocalDraftThread: _isLocalDraftThread, + onNewTaskGraph, forceExpandedOnMobile, projectSelectionRequired, phase, @@ -7662,6 +7666,25 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ) : null} + {onNewTaskGraph ? ( + + event.preventDefault()} + onClick={onNewTaskGraph} + aria-label="New task graph" + /> + } + > + + + New task graph + + ) : null} import("./TaskGraphEditor"); +const TaskGraphEditor = lazy(loadTaskGraphEditor); + +const EMPTY_GRAPHS: ReadonlyArray = []; + +/** Finished graphs hidden from the chat for the rest of this session. */ +const useDismissedTaskGraphs = create<{ + readonly ids: ReadonlySet; + readonly dismiss: (id: string) => void; +}>((set) => ({ + ids: new Set(), + dismiss: (id) => set((state) => ({ ids: new Set(state.ids).add(id) })), +})); + +const openExternal = (url: string) => void readLocalApi()?.shell.openExternal(url); + +/** + * The task graphs a thread proposed, docked above its composer, each drawn as + * a left-to-right diagram. Any graph collapses to a summary line, and finished + * ones can be dismissed. Fed by the live subscription, so status needs no refresh. + */ +export function TaskGraphCards(props: { + readonly environmentId: EnvironmentId; + readonly threadId: ThreadId; +}) { + const graphsQuery = useEnvironmentQuery( + serverEnvironment.taskGraphsLive({ + environmentId: props.environmentId, + input: { threadId: props.threadId }, + }), + ); + const dismissed = useDismissedTaskGraphs((state) => state.ids); + const dismiss = useDismissedTaskGraphs((state) => state.dismiss); + // A load error renders nothing: servers from before task graphs reject the + // subscription, and every thread on them would otherwise carry an error row. + const graphs = (graphsQuery.data?.graphs ?? EMPTY_GRAPHS).filter( + (graph) => !(dismissed.has(graph.id) && isTaskGraphFinished(graph.status)), + ); + return graphs.map((graph) => ( + dismiss(graph.id)} + /> + )); +} + +function TaskGraphCard(props: { + readonly environmentId: EnvironmentId; + readonly graph: TaskGraph; + readonly onDismiss: () => void; +}) { + const { graph } = props; + const finished = isTaskGraphFinished(graph.status); + // Expanded by default, finished or not: a graph that ran on its own should still show its shape. + const [expanded, setExpanded] = useState(true); + const [editorOpenLocally, setEditorOpen] = useState(false); + // A graph just started from the composer opens straight into its editor. + const requestedNodeKey = useTaskGraphEditorRequest((state) => + state.graphId === graph.id ? state.nodeKey : null, + ); + const clearEditorRequest = useTaskGraphEditorRequest((state) => state.clear); + const editorOpen = editorOpenLocally || requestedNodeKey !== null; + const [editorNodeKey, setEditorNodeKey] = useState(null); + const openEditor = (nodeKey: string | null) => { + setEditorNodeKey(nodeKey); + setEditorOpen(true); + }; + const commands = useTaskGraphCommands(props.environmentId, graph); + const labels = useTaskGraphLabels(props.environmentId, graph); + const listId = useId(); + const pullRequests = taskGraphPullRequestLinks(graph.nodes); + + return ( + + + + + + + + {graph.title} + + + {TASK_GRAPH_STATUS_LABEL[graph.status]} · {taskGraphProgressLabel(graph)} + + + + {taskGraphResourcesLabel(graph, labels.modelLabel)} + + {!expanded + ? pullRequests.map((pullRequest) => ( + + )) + : null} + + + {graph.status === "draft" ? ( + + ) : null} + + void loadTaskGraphEditor()} + onFocus={() => void loadTaskGraphEditor()} + onClick={() => openEditor(null)} + /> + } + > + + + {graph.status === "draft" ? "Edit graph" : "Open graph"} + + {!finished ? ( + + void commands.cancel()} + /> + } + > + + + + {graph.status === "draft" ? "Discard draft" : "Cancel graph"} + + + ) : null} + + {finished ? ( + + ) : null} + + + {commands.error !== null ? ( + +

+ {commands.error} +

+ + ) : null} + + {expanded ? ( + { + void loadTaskGraphEditor(); + openEditor(key); + }} + /> + ) : null} + + + { + setEditorOpen(open); + if (!open && requestedNodeKey !== null) clearEditorRequest(); + }} + > + + {editorOpen ? ( + + + + } + > + + + ) : null} + + + + ); +} + +/** + * The editor for a graph drafted on a new chat. Nothing reaches the server + * until Run, which creates the chat's thread and the graph together; closing + * keeps the draft for as long as the new chat stays open. + */ +export function DraftTaskGraphDialog(props: { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + readonly threadId: ThreadId; + readonly baseRef: string | null; + readonly modelSelection: ModelSelection | null; + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; + readonly onRun: (nodes: ReadonlyArray) => Promise; +}) { + const draft = useDraftTaskGraph({ + canRun: true, + onRun: props.onRun, + onDiscard: () => props.onOpenChange(false), + }); + const [createdAt] = useState(() => new Date().toISOString()); + const graph: TaskGraph = { + id: TaskGraphId.make("draft"), + projectId: props.projectId, + threadId: props.threadId, + title: "New task graph", + baseRef: props.baseRef ?? "HEAD", + modelSelection: props.modelSelection, + status: "draft", + nodes: draft.nodes, + createdAt, + updatedAt: createdAt, + }; + const labels = useTaskGraphLabels(props.environmentId, graph); + return ( + + + {props.open ? ( + + + + } + > + + + ) : null} + + + ); +} diff --git a/apps/web/src/components/chat/TaskGraphDiagram.tsx b/apps/web/src/components/chat/TaskGraphDiagram.tsx new file mode 100644 index 000000000000..794f42db1d46 --- /dev/null +++ b/apps/web/src/components/chat/TaskGraphDiagram.tsx @@ -0,0 +1,287 @@ +import type { EnvironmentId, TaskGraphNode } from "@t3tools/contracts"; +import { taskGraphLayers } from "@t3tools/shared/taskGraph"; +import { Link } from "@tanstack/react-router"; +import { CornerDownRightIcon, FolderIcon, GitBranchIcon } from "lucide-react"; +import { type ReactNode, useEffect, useRef } from "react"; + +import { cn } from "~/lib/utils"; +import { usePrimarySettings } from "../../hooks/useSettings"; +import { formatUpcomingTimestamp } from "../../timestampFormat"; +import { PreviewCard, PreviewCardPopup, PreviewCardTrigger } from "../ui/preview-card"; +import { + pullRequestLabel, + TASK_GRAPH_DIAGRAM, + TASK_GRAPH_NODE_STATUS_DOT_CLASS, + TASK_GRAPH_NODE_STATUS_LABEL, + taskGraphDiagramLayout, + taskGraphDiagramScrollLeft, + taskGraphNodeStatusText, + taskGraphNodeWaitDetail, + taskGraphWorkspaceLabel, +} from "./taskGraphView"; +import type { TaskGraphLabels } from "./useTaskGraphLabels"; + +/** + * The graph as boxes and curves, left to right, for the card above the + * composer. Static: status changes recolour dots, nothing animates. Once the + * first columns are done, the card scrolls to where work is left. + */ +export function TaskGraphDiagram(props: { + readonly environmentId: EnvironmentId; + readonly nodes: ReadonlyArray; + readonly labels: TaskGraphLabels; + readonly onOpenPullRequest: (url: string) => void; + /** Opens the editor on a task; bound to double-click and Enter on its box. */ + readonly onEditNode: (key: string) => void; +}) { + const layers = taskGraphLayers(props.nodes); + const layout = taskGraphDiagramLayout(layers); + const scrollLeft = taskGraphDiagramScrollLeft(layers); + const rootRef = useRef(null); + // Runs only when the leading column changes, so status updates never undo a manual scroll. + useEffect(() => { + rootRef.current?.closest('[data-slot="scroll-area-viewport"]')?.scrollTo({ left: scrollLeft }); + }, [scrollLeft, rootRef]); + const titles = new Map(props.nodes.map((node) => [node.key, node.title])); + const titleOf = (key: string) => titles.get(key) ?? key; + const timestampFormat = usePrimarySettings((settings) => settings.timestampFormat); + const formatTime = (iso: string) => formatUpcomingTimestamp(iso, timestampFormat); + + return ( +
+ + {layout.edges.map((edge) => ( + + ))} + + {layout.nodes.map(({ node, x, y }) => ( + props.onEditNode(node.key)} + /> + ))} +
+ ); +} + +function TaskGraphDiagramNode(props: { + readonly environmentId: EnvironmentId; + readonly node: TaskGraphNode; + readonly labels: TaskGraphLabels; + readonly x: number; + readonly y: number; + readonly onOpenPullRequest: (url: string) => void; + readonly titleOf: (key: string) => string; + /** Renders an upcoming instant, such as when a wait ends. */ + readonly formatTime: (iso: string) => string; + readonly onEdit: () => void; +}) { + const { node } = props; + const pullRequestUrl = node.pullRequestResult?.url ?? null; + const machine = props.labels.nodeMachineLabel(node); + const status = taskGraphNodeStatusText(node, props.formatTime); + const title = {node.title}; + const box = ( +
{ + if (event.key === "Enter" && event.target === event.currentTarget) props.onEdit(); + }} + className={cn( + "absolute flex cursor-default select-none flex-col justify-center gap-0.5 rounded-md border bg-background px-2 text-xs outline-none hover:border-ring/60 focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ring", + node.status === "failed" ? "border-destructive/50" : "border-border", + (node.status === "skipped" || node.status === "cancelled") && "opacity-60", + )} + style={{ + left: props.x, + top: props.y, + width: TASK_GRAPH_DIAGRAM.nodeWidth, + height: TASK_GRAPH_DIAGRAM.nodeHeight, + }} + > +
+ + {title} +
+
+ + {status} · {machine === "Auto" ? machine : `on ${machine}`} + + {pullRequestUrl !== null ? ( + + ) : null} +
+
+ {props.labels.nodeModelLabel(node)} + +
+
+ ); + return ( + + + + + + + ); +} + +/** What a hovered box expands to: the task itself, what it waits on, and how it went. */ +function TaskGraphNodeDetails(props: { + readonly environmentId: EnvironmentId; + readonly node: TaskGraphNode; + readonly labels: TaskGraphLabels; + readonly titleOf: (key: string) => string; + readonly formatTime: (iso: string) => string; +}) { + const { node } = props; + const pullRequestError = + node.pullRequestResult?.status === "failed" ? node.pullRequestResult.error : null; + const wait = taskGraphNodeWaitDetail(node, props.formatTime); + return ( +
+
+ + {node.title} + + {TASK_GRAPH_NODE_STATUS_LABEL[node.status]} + +
+

+ {node.prompt} +

+ {wait !== null ? {wait} : null} + {node.dependsOn.length > 0 ? ( + + {node.dependsOn.map(props.titleOf).join(", ")} + + ) : null} + {props.labels.nodeModelLabel(node)} + + {props.labels.nodeMachineLabel(node)} + + + {taskGraphWorkspaceLabel(node, props.titleOf)} + + {node.branch !== null ? ( + + {node.branch} + + ) : null} + {node.summary !== null ? ( + + {node.summary} + + ) : null} + {node.error !== null ? ( +

{node.error}

+ ) : null} + {pullRequestError !== null ? ( +

+ Pull request failed: {pullRequestError} +

+ ) : null} +
+ Double-click to edit + {node.threadId !== null ? ( + + Open thread + + ) : null} +
+
+ ); +} + +/** The box's workspace hint: an icon, plus the dependency's name when it continues one. */ +function TaskGraphWorkspaceBadge(props: { + readonly node: TaskGraphNode; + readonly titleOf: (key: string) => string; +}) { + const { node } = props; + const dependency = node.dependsOn[0]; + if (node.workspace === "dependency" && dependency !== undefined) { + return ( + + + Continues + {props.titleOf(dependency)} + + ); + } + const Icon = node.workspace === "root" ? FolderIcon : GitBranchIcon; + return ( + + ); +} + +function TaskGraphNodeDetail(props: { readonly label: string; readonly children: ReactNode }) { + return ( +
+ {props.label} + {props.children} +
+ ); +} diff --git a/apps/web/src/components/chat/TaskGraphEditor.css b/apps/web/src/components/chat/TaskGraphEditor.css new file mode 100644 index 000000000000..fa980dbcb4b7 --- /dev/null +++ b/apps/web/src/components/chat/TaskGraphEditor.css @@ -0,0 +1,13 @@ +/* + * xyflow draws edges, handles, the connection line and the dot grid itself, + * so its theme variables map to ours here instead of in classes on elements + * we render. Loaded with the editor chunk only. + */ +.react-flow[data-task-graph-canvas] { + --xy-edge-stroke: var(--muted-foreground); + --xy-edge-stroke-selected: var(--ring); + --xy-edge-stroke-width: 1.5; + --xy-connectionline-stroke: var(--ring); + --xy-handle-background-color: var(--muted-foreground); + --xy-background-pattern-dots-color: var(--border); +} diff --git a/apps/web/src/components/chat/TaskGraphEditor.tsx b/apps/web/src/components/chat/TaskGraphEditor.tsx new file mode 100644 index 000000000000..13b1fb37e756 --- /dev/null +++ b/apps/web/src/components/chat/TaskGraphEditor.tsx @@ -0,0 +1,1174 @@ +import "@xyflow/react/dist/base.css"; +import "./TaskGraphEditor.css"; + +import type { + EnvironmentId, + ModelSelection, + TaskGraph, + TaskGraphNode, + TaskGraphNodeWorkspace, +} from "@t3tools/contracts"; +import { + isActiveTaskGraphNodeStatus, + isUnstartedTaskGraphNode, + isTerminalTaskGraphNodeStatus, + taskGraphNodeOpensPullRequest, +} from "@t3tools/shared/taskGraph"; +import { + localSnoozeDate, + localSnoozeTime, + resolveCustomSnooze, +} from "@t3tools/client-runtime/state/thread-settled"; +import { Link } from "@tanstack/react-router"; +import { + Background, + Handle, + Position, + ReactFlow, + useReactFlow, + type Connection, + type NodeProps, +} from "@xyflow/react"; +import { + CalendarIcon, + MessageSquareTextIcon, + PlayIcon, + PlusIcon, + RotateCcwIcon, + ScissorsIcon, + SquareIcon, + Trash2Icon, + UnlinkIcon, +} from "lucide-react"; +import { useEffect, useId, useMemo, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { readLocalApi } from "~/localApi"; +import { usePrimarySettings } from "../../hooks/useSettings"; +import { getCustomModelOptionsByInstance } from "../../modelSelection"; +import { NO_PROVIDER_MODEL_SELECTION } from "../../providerInstances"; +import { formatUpcomingTimestamp, weekStartsOn } from "../../timestampFormat"; +import { Button } from "../ui/button"; +import { Calendar } from "../ui/calendar"; +import { DialogDescription, DialogHeader, DialogTitle } from "../ui/dialog"; +import { Input } from "../ui/input"; +import { Label } from "../ui/label"; +import { Popover, PopoverPopup, PopoverTrigger } from "../ui/popover"; +import { ScrollArea } from "../ui/scroll-area"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { Textarea } from "../ui/textarea"; +import { Toggle, ToggleGroup } from "../ui/toggle-group"; +import { PullRequestGlyph } from "../pullRequest/pullRequestIcons"; +import { ProviderModelPicker } from "./ProviderModelPicker"; +import { + layoutTaskGraph, + taskGraphCanvasEdgeId, + taskGraphCanvasElements, + taskGraphLayoutSignature, + type TaskGraphCanvasNode, + type TaskGraphCanvasPoint, +} from "./taskGraphCanvasLayout"; +import { + canContinueTaskGraphWorktree, + isTaskGraphFinished, + pullRequestLabel, + TASK_GRAPH_NODE_STATUS_DOT_CLASS, + TASK_GRAPH_NODE_STATUS_LABEL, + TASK_GRAPH_STATUS_LABEL, + type TaskGraphJoinMachine, + taskGraphJoinMachines, + taskGraphNodeKeyFromTitle, + taskGraphNodeWaitDetail, + taskGraphProgressLabel, +} from "./taskGraphView"; +import type { TaskGraphCommands } from "./useTaskGraphCommands"; +import { TASK_GRAPH_PEER_STATUS_HINT, type TaskGraphLabels } from "./useTaskGraphLabels"; + +type Selection = + | { readonly kind: "none" } + | { readonly kind: "node"; readonly key: string } + | { readonly kind: "edge"; readonly dependency: string; readonly dependent: string }; + +const NO_SELECTION: Selection = { kind: "none" }; + +// One editor is open at a time; keeping its last layout lets status updates +// from a running graph skip dagre until nodes or dependencies change. +let cachedLayout: { + readonly signature: string; + readonly positions: ReadonlyMap; +} | null = null; + +function graphLayout(nodes: ReadonlyArray) { + const signature = taskGraphLayoutSignature(nodes); + if (cachedLayout?.signature !== signature) { + cachedLayout = { signature, positions: layoutTaskGraph(nodes) }; + } + return cachedLayout; +} + +const openExternal = (url: string) => void readLocalApi()?.shell.openExternal(url); + +function TaskCanvasNode({ data, selected }: NodeProps) { + const { node, opensPullRequest } = data; + return ( +
+ + + + {node.title} + + + {node.key} + {TASK_GRAPH_NODE_STATUS_LABEL[node.status]} + {opensPullRequest ? ( + + ) : null} + + +
+ ); +} + +const NODE_TYPES = { task: TaskCanvasNode }; + +/** Fits the view on mount; keyed by layout so it refits on structure changes, not status updates. */ +function FitViewOnMount() { + const { fitView } = useReactFlow(); + useEffect(() => { + void fitView({ maxZoom: 1 }); + }, [fitView]); + return null; +} + +/** + * Canvas editor for one task graph, loaded only when the card opens it. The + * subscription's graph is the only copy: every change is sent as an edit and + * the canvas redraws from what the server reports back. + */ +export default function TaskGraphEditor(props: { + readonly environmentId: EnvironmentId; + readonly graph: TaskGraph; + readonly commands: TaskGraphCommands; + readonly labels: TaskGraphLabels; + /** Task to select on open, such as the box double-clicked in the chat diagram. */ + readonly initialNodeKey?: string | null; +}) { + const { graph, commands } = props; + const [selectionState, setSelection] = useState(() => + props.initialNodeKey == null ? NO_SELECTION : { kind: "node", key: props.initialNodeKey }, + ); + const layout = graphLayout(graph.nodes); + const selection = resolveSelection(selectionState, graph); + const { nodes, edges } = taskGraphCanvasElements(graph.nodes, layout.positions, { + nodeKey: selection.kind === "node" ? selection.key : null, + edgeId: + selection.kind === "edge" + ? taskGraphCanvasEdgeId(selection.dependency, selection.dependent) + : null, + }); + const canEdit = commands.canEdit && !commands.busy; + const finished = isTaskGraphFinished(graph.status); + + const unstartedNode = (key: string) => + graph.nodes.find((node) => node.key === key && isUnstartedTaskGraphNode(node)); + + const connect = (connection: Connection) => { + const dependent = unstartedNode(connection.target); + if (dependent === undefined || dependent.dependsOn.includes(connection.source)) return; + void commands.edit([ + { + type: "update_node", + key: dependent.key, + dependsOn: [...dependent.dependsOn, connection.source], + }, + ]); + }; + + // Adds a placeholder task to the graph and selects it for editing: after the + // selected task, continuing its worktree where it can, or with nothing + // selected, as the parent of the unstarted tasks at the start (a started + // task can no longer be made to wait). + const addTask = async () => { + const after = selection.kind === "node" ? selection.key : null; + const parent = graph.nodes.find((node) => node.key === after); + const key = taskGraphNodeKeyFromTitle( + "New task", + graph.nodes.map((node) => node.key), + ); + const children = + after === null + ? graph.nodes.filter( + (node) => node.dependsOn.length === 0 && isUnstartedTaskGraphNode(node), + ) + : []; + const accepted = await commands.edit([ + { + type: "add_node", + node: { + key, + title: "New task", + prompt: "Describe what this task should do.", + dependsOn: after === null ? [] : [after], + ...(parent !== undefined && canContinueTaskGraphWorktree(graph.nodes, parent) + ? { workspace: "dependency" as const } + : {}), + }, + }, + ...children.map((child) => ({ + type: "update_node" as const, + key: child.key, + dependsOn: [key], + })), + ]); + if (accepted) setSelection({ kind: "node", key }); + }; + + return ( + <> + +
+
+ {graph.title} + + {TASK_GRAPH_STATUS_LABEL[graph.status]} · {taskGraphProgressLabel(graph)} · from{" "} + {graph.baseRef} + +
+
+ + {!finished ? ( + + ) : null} + {graph.status === "draft" ? ( + + ) : null} +
+
+ {commands.error !== null ? ( +

+ {commands.error} +

+ ) : null} + +
+
+ + connection.source !== connection.target && + unstartedNode(connection.target) !== undefined + } + onConnect={connect} + onNodeClick={(_event, node) => setSelection({ kind: "node", key: node.id })} + onEdgeClick={(_event, edge) => + setSelection({ kind: "edge", dependency: edge.source, dependent: edge.target }) + } + onPaneClick={() => setSelection(NO_SELECTION)} + data-task-graph-canvas + > + + + +
+
+ +
+ +
+ +
+
+ + ); +} + +/** Drops a selection the server has since removed, such as a deleted node. */ +function resolveSelection(selection: Selection, graph: TaskGraph): Selection { + if (selection.kind === "node") { + return graph.nodes.some((node) => node.key === selection.key) ? selection : NO_SELECTION; + } + if (selection.kind === "edge") { + const dependent = graph.nodes.find((node) => node.key === selection.dependent); + return dependent?.dependsOn.includes(selection.dependency) ? selection : NO_SELECTION; + } + return selection; +} + +function SelectionPanel(props: { + readonly environmentId: EnvironmentId; + readonly graph: TaskGraph; + readonly commands: TaskGraphCommands; + readonly labels: TaskGraphLabels; + readonly selection: Selection; + readonly onSelect: (selection: Selection) => void; +}) { + const { graph, commands, selection } = props; + const canEdit = commands.canEdit && !commands.busy; + + if (selection.kind === "edge") { + const dependent = graph.nodes.find((node) => node.key === selection.dependent); + const dependency = graph.nodes.find((node) => node.key === selection.dependency); + if (dependent === undefined || dependency === undefined) return null; + return ( +
+

Dependency

+

+ {dependent.title} starts after{" "} + {dependency.title} succeeds. +

+ {isUnstartedTaskGraphNode(dependent) ? ( + + ) : ( +

+ {dependent.title} has started, so its dependencies are fixed. +

+ )} +
+ ); + } + + if (selection.kind === "node") { + const node = graph.nodes.find((candidate) => candidate.key === selection.key); + if (node === undefined) return null; + return ( + + ); + } + + return ( +
+

Graph

+

+ Each task runs as its own thread, in its own worktree unless set otherwise. Drag from the + right edge of a task to the left edge of another to make the second wait for the first. +

+

Tasks that nothing depends on open a pull request unless you turn it off.

+

Select a task or dependency to change it.

+

+ Add task puts the new task after the selected one, continuing its worktree. With nothing + selected, it becomes the parent of the tasks at the start. +

+
+ ); +} + +// Dependency choices carry a prefix, so no task key (keys are slugs) can equal the "any machine" value. +const JOIN_AUTO = "any"; +const joinValue = (dependencyKey: string) => `dependency:${dependencyKey}`; + +/** + * Where a task that merges branches from several machines brings them + * together: on one dependency's machine, continuing its worktree and pulling + * the other branches in through origin, or in a new worktree wherever load + * balancing puts it. + */ +function TaskJoinMachineField(props: { + readonly node: TaskGraphNode; + readonly options: ReadonlyArray; + readonly labels: TaskGraphLabels; + readonly disabled: boolean; + readonly titleOf: (key: string) => string; + readonly onChoose: (dependencyKey: string | null) => void; +}) { + const id = useId(); + const continued = + props.node.workspace === "dependency" ? (props.node.dependsOn[0] ?? null) : null; + const current = continued === null ? JOIN_AUTO : joinValue(continued); + const optionLabel = (option: TaskGraphJoinMachine) => + `${props.labels.machineLabel(option.machine)}, continuing ${option.dependency.title}`; + const chosen = props.options.find((option) => option.dependency.key === continued); + // The continued dependency is left out of the options while its machine is unknown, + // such as when it is balanced automatically and has not started yet. + const pendingLabel = + continued !== null && chosen === undefined + ? `Machine not known yet, continuing ${props.titleOf(continued)}` + : null; + return ( +
+ + +

+ The branches this task merges are on different machines. The other branches are pushed and + pulled in through origin. +

+
+ ); +} + +function NodePanel(props: { + readonly environmentId: EnvironmentId; + readonly graph: TaskGraph; + readonly node: TaskGraphNode; + readonly commands: TaskGraphCommands; + readonly labels: TaskGraphLabels; +}) { + const { graph, node, commands } = props; + const placement: TaskNodePlacementContext = { + graph, + labels: props.labels, + dependency: graph.nodes.find((candidate) => candidate.key === node.dependsOn[0]) ?? null, + }; + const canEdit = commands.canEdit && !commands.busy; + const opensByDefault = taskGraphNodeOpensPullRequest(graph.nodes, { + ...node, + pullRequest: null, + }); + const canCancelBranch = + isActiveTaskGraphNodeStatus(node.status) || + node.status === "waiting" || + (node.status === "pending" && graph.status !== "draft"); + const canRetry = isTerminalTaskGraphNodeStatus(node.status) && node.status !== "succeeded"; + const timestampFormat = usePrimarySettings((settings) => settings.timestampFormat); + const wait = taskGraphNodeWaitDetail(node, (iso) => + formatUpcomingTimestamp(iso, timestampFormat), + ); + + const actions = ( +
+ {canCancelBranch ? ( + + ) : null} + {canRetry ? ( + + ) : null} +
+ ); + + const joinMachines = taskGraphJoinMachines(graph.nodes, node); + + if (isUnstartedTaskGraphNode(node)) { + return ( +
+ {wait !== null ?

{wait}

: null} + {joinMachines !== null ? ( + graph.nodes.find((entry) => entry.key === key)?.title ?? key} + onChoose={(dependencyKey) => + void commands.edit([ + { + type: "update_node", + key: node.key, + ...(dependencyKey === null + ? { workspace: "worktree" } + : { + dependsOn: [ + dependencyKey, + ...node.dependsOn.filter((key) => key !== dependencyKey), + ], + workspace: "dependency", + // Continuing a worktree runs where it is, so a pin no longer applies. + environmentId: null, + }), + }, + ]) + } + /> + ) : null} + { + await commands.edit([ + { + type: "update_node", + key: node.key, + ...(values.title === node.title ? {} : { title: values.title }), + ...(values.prompt === node.prompt ? {} : { prompt: values.prompt }), + ...(values.pullRequest === node.pullRequest + ? {} + : { pullRequest: values.pullRequest }), + ...(sameModel(values.modelSelection, node.modelSelection) + ? {} + : { modelSelection: values.modelSelection }), + ...(values.environmentId === node.environmentId + ? {} + : { environmentId: values.environmentId }), + ...(values.workspace === node.workspace ? {} : { workspace: values.workspace }), + ...(values.startAt === node.startAt ? {} : { startAt: values.startAt }), + }, + ]); + }} + /> +
+ + {actions} +
+
+ ); + } + + const pullRequest = node.pullRequestResult; + const pullRequestUrl = pullRequest?.url ?? null; + return ( +
+
+

{node.title}

+

+ + {TASK_GRAPH_NODE_STATUS_LABEL[node.status]} + {node.key} +

+
+ {wait !== null ?

{wait}

: null} + {node.error !== null ?

{node.error}

: null} + {node.branch !== null ? ( +

{node.branch}

+ ) : null} +
+ {node.threadId !== null ? ( + + ) : null} + {pullRequestUrl !== null ? ( + + ) : null} +
+ {pullRequest?.status === "failed" && pullRequest.error !== null ? ( +

Pull request failed: {pullRequest.error}

+ ) : null} + {actions} + +
+

Prompt

+

{node.prompt}

+
+ {node.summary !== null ? ( +
+

Result

+

{node.summary}

+
+ ) : null} +
+ ); +} + +interface TaskNodeValues { + readonly title: string; + readonly prompt: string; + /** Null follows the default: a pull request only when nothing depends on the task. */ + readonly pullRequest: boolean | null; + /** Null runs with the graph's model. */ + readonly modelSelection: ModelSelection | null; + /** Null balances across machines when the task starts. */ + readonly environmentId: EnvironmentId | null; + readonly workspace: TaskGraphNodeWorkspace; + /** Earliest start as an ISO time; null starts as soon as its dependencies succeed. */ + readonly startAt: string | null; +} + +type TaskNodePlacement = Pick; + +/** What the placement fields resolve names against; `dependency` is the task's first dependency. */ +interface TaskNodePlacementContext { + readonly graph: TaskGraph; + readonly labels: TaskGraphLabels; + readonly dependency: TaskGraphNode | null; +} + +const sameModel = (left: ModelSelection | null, right: ModelSelection | null) => + left === right || + (left !== null && + right !== null && + left.instanceId === right.instanceId && + left.model === right.model); + +const PULL_REQUEST_CHOICES = ["auto", "on", "off"] as const; +type PullRequestChoice = (typeof PULL_REQUEST_CHOICES)[number]; + +const toChoice = (value: boolean | null): PullRequestChoice => + value === null ? "auto" : value ? "on" : "off"; +const fromChoice = (choice: PullRequestChoice): boolean | null => + choice === "auto" ? null : choice === "on"; + +/** Form text stays local until submitted; everything else comes from the server. */ +function TaskNodeForm(props: { + readonly heading: string; + readonly initial: TaskNodeValues; + readonly placement: TaskNodePlacementContext; + readonly opensByDefault: boolean; + readonly disabled: boolean; + readonly submitLabel: string; + readonly onSubmit: (values: TaskNodeValues) => Promise; +}) { + const id = useId(); + const [title, setTitle] = useState(props.initial.title); + const [prompt, setPrompt] = useState(props.initial.prompt); + const [pullRequest, setPullRequest] = useState(props.initial.pullRequest); + const [placement, setPlacement] = useState(props.initial); + const [start, setStart] = useState(() => initialStartChoice(props.initial.startAt)); + const startAt = resolveStartAt(start, props.initial.startAt); + const values: TaskNodeValues = { + title: title.trim(), + prompt: prompt.trim(), + pullRequest, + modelSelection: placement.modelSelection, + environmentId: placement.environmentId, + workspace: placement.workspace, + startAt, + }; + const dirty = + values.title !== props.initial.title || + values.prompt !== props.initial.prompt || + values.pullRequest !== props.initial.pullRequest || + !sameModel(values.modelSelection, props.initial.modelSelection) || + values.environmentId !== props.initial.environmentId || + values.workspace !== props.initial.workspace || + values.startAt !== props.initial.startAt; + const startAtInvalid = start.scheduled && startAt === null; + const valid = + values.title.length > 0 && + values.title.length <= 120 && + values.prompt.length > 0 && + !startAtInvalid; + + return ( +
{ + event.preventDefault(); + if (dirty && valid && !props.disabled) void props.onSubmit(values); + }} + > +

{props.heading}

+
+ + setTitle(event.target.value)} + /> +
+
+ +