From bbb0b43c9ceacd98a3304f4237bbf29f830b45fc Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Sat, 11 Jul 2026 23:39:01 -0700 Subject: [PATCH 1/3] fix(workflows): preserve stage duration across durable resume Persist pause-adjusted active-stage timing at repeated checkpoints and rebase resumed stage clocks from the durable baseline. Cover file and DBOS hydration, process-boundary completion, pause accounting, and replay idempotency. Assistant-model: GPT-5.6 Sol --- packages/coding-agent/docs/workflows.md | 2 +- packages/workflows/CHANGELOG.md | 4 + packages/workflows/README.md | 2 +- packages/workflows/src/durable/backend.ts | 23 ++- .../workflows/src/durable/scoped-backend.ts | 7 +- .../workflows/src/durable/stage-primitive.ts | 30 +++- .../runs/foreground/executor-stage-call.ts | 4 +- .../runs/foreground/stage-runner-options.ts | 1 + packages/workflows/src/shared/timing.ts | 4 + packages/workflows/src/shared/types.ts | 2 + test/unit/durable-backend.test.ts | 19 +++ test/unit/durable-dbos-backend.test.ts | 9 +- .../unit/durable-stage-session-resume.test.ts | 154 +++++++++++++++++- 13 files changed, 239 insertions(+), 22 deletions(-) diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index b93bc695d..ca1a04591 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -410,7 +410,7 @@ Named runs go to the background. Common controls: When a paused stage is resumed with a message, Atomic lets the stage answer that resume message, then (if the stage has not already finalized) injects `Continue where you left off. If you believe you are finished with your original task (or a redefined task if the user told you), stop.` into the same stage session before normal stage completion/readiness handling. This keeps interrupted work moving without asking you to manually type a second continuation prompt while also preventing stages that already finished their scoped work from overstepping. -Durable `/workflow resume` preserves completed stage metadata and graph topology. When it reopens an unfinished stage session mid-chat, it uses the same continuation prompt shown above. Replayed `ctx.stage`, `ctx.task`, `ctx.chain`, `ctx.parallel`, and child-workflow checkpoints keep their original summaries, timing, session/model metadata, and parallel fanout parentage in status and graph views instead of appearing as freshly flattened replay nodes. +Durable `/workflow resume` preserves completed stage metadata, active-stage elapsed time, and graph topology. While an LM stage is active, repeated durable checkpoints refresh its accumulated pause-adjusted duration even when its session file does not change. When a new Atomic process reopens that unfinished session mid-chat, timing continues from the saved baseline and uses the same continuation prompt shown above, so status, graph, stored, and lifecycle duration includes both pre-resume and post-resume work without double-counting pauses. Replayed `ctx.stage`, `ctx.task`, `ctx.chain`, `ctx.parallel`, and child-workflow checkpoints keep their original summaries, timing, session/model metadata, and parallel fanout parentage instead of appearing as freshly flattened replay nodes. Workflow stage sessions are marked as **internal** and excluded from the standard `/resume`, `atomic -r`, and `--continue` history so they do not clutter your interactive session picker. They remain resumable and inspectable through the workflow-specific commands and tool actions shown here (`/workflow resume`, `/workflow attach`, `workflow({ action: "status" | "stages" | "stage" | "resume" })`), which read the run/stage store and its `sessionFile` links directly. Passing a stage session's file path to `--session` still opens it explicitly. Legacy workflow sessions created before this marker behavior lack the signal and will continue to appear in the standard history until they age out. diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 1bdbbfeb3..9e42ceb0c 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -86,6 +86,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Restored Ralph's builtin implementation-stage prompts to require subagent-led investigation, editing, and validation, reversing the selective direct-implementation wording introduced with intent-first routing. - Expanded model-facing workflow guidance to treat composition as a first-class design option: custom parents can import reusable project/package workflows or bundled builtin definitions, invoke them through `ctx.workflow(...)`, and nest further child workflows within `maxDepth` while preserving expanded graph visibility, HIL, durability, controls, and declared output contracts. +### Fixed + +- Fixed durable mid-stage workflow resume to refresh pause-adjusted active-stage duration at repeated session checkpoints, preserve it through file and DBOS hydration, and continue accumulating it after a process-boundary resume without changing replay identity or re-running completed side effects ([#1713](https://github.com/bastani-inc/atomic/issues/1713)). + ## [0.9.5] - 2026-07-11 ### Breaking Changes diff --git a/packages/workflows/README.md b/packages/workflows/README.md index ffd6cd285..f4310e386 100644 --- a/packages/workflows/README.md +++ b/packages/workflows/README.md @@ -544,7 +544,7 @@ Input overrides are bare `key=value` tokens (no leading `--`). Values are JSON-p Named workflow launches always run as **background tasks** in interactive sessions. Model-launched direct `task`, `tasks`, and `chain` calls must set top-level `async: true` so the chat editor stays free; inspection and control calls are unaffected. Foreground launches are reserved for explicit user requests or technical requirements, with notice before launch. Press **F2** (or `/workflow connect `) to attach to the live graph viewer; HIL prompts (`ctx.ui.input/confirm/select/editor/custom`) appear as awaiting-input graph nodes. Press Enter on a focused node, or click a visible graph node directly, to open that stage and answer locally, never as a modal dialog over the chat. Attached stage chats capture mouse/trackpad wheel events by default so scrolling stays inside the active stage transcript or prompt instead of falling through to terminal/main-chat scrollback. Press `ctrl+t` to toggle **copy mode**: copy mode disables workflow-chat mouse reporting so normal terminal/tmux text selection can work; press `ctrl+t` again to leave copy mode and restore workflow-chat scrolling. Archived read-only stage transcripts show the same copy-mode footer/status, allowing their transcript text to be selected and copied while preserving `esc` close and `ctrl+d` graph navigation. While copy mode is on, wheel/trackpad gestures are handled by the terminal/tmux and may scroll terminal scrollback, so leave copy mode before using the wheel again. Human input is detected when those runtime `ctx.ui.*` calls execute; workflows no longer have a declaration-time HIL flag. -Durable resume preserves the original completed-stage shape: replayed stage/task/parallel/child checkpoints retain summaries, durations, session/model metadata, and parallel branch parentage in the graph instead of flattening fanout branches into a sequential replay chain. +Durable resume preserves both completed and active stage timing. Replayed stage/task/parallel/child checkpoints retain summaries, durations, session/model metadata, and parallel branch parentage in the graph instead of flattening fanout branches into a sequential replay chain. While an LM stage is active, repeated durable session checkpoints refresh its pause-adjusted accumulated duration even when the session file is unchanged; a later `/workflow resume` continues from that baseline, so status, graph, stored, and lifecycle durations include the elapsed segments from before and after the process boundary without counting paused time twice. Nested `ctx.workflow(...)` calls are displayed as an expanded graph within the top-level run. `/workflow status` and run pickers list only top-level user-launched workflows, not implementation-owned child runs. The `workflow` tool's `stages`, `stage`, `transcript`, `send`, `pause`, `interrupt`, and `resume` actions can still target visible child stage ids, prefixes, or names from the expanded graph; Atomic routes the control action to the owning nested run internally. (`stages`, `stage`, `transcript`, and `send` are `workflow` tool actions, not `/workflow` slash subcommands; the slash command exposes `connect`, `attach`, `pause`, `list`, `status`, `interrupt`, `kill`, `resume`, `reload`, and `inputs`.) diff --git a/packages/workflows/src/durable/backend.ts b/packages/workflows/src/durable/backend.ts index a01f9fb22..dc2505a2c 100644 --- a/packages/workflows/src/durable/backend.ts +++ b/packages/workflows/src/durable/backend.ts @@ -94,8 +94,13 @@ export interface DurableWorkflowBackend { /** Look up a cached stage output by replay key. */ getStageOutput(workflowId: string, replayKey: string): WorkflowSerializableValue | undefined; - /** Look up resumable stage session metadata by replay key, when available. */ - getStageSession(workflowId: string, replayKey: string): { sessionId?: string; sessionFile?: string } | undefined; + /** Look up the latest resumable stage session and accumulated active timing. */ + getStageSession(workflowId: string, replayKey: string): { + sessionId?: string; + sessionFile?: string; + startedAt?: number; + durationMs?: number; + } | undefined; /** List all checkpoints for a workflow (in completion order). */ listCheckpoints(workflowId: string): readonly DurableCheckpoint[]; @@ -246,10 +251,20 @@ export class InMemoryDurableBackend implements DurableWorkflowBackend { return checkpoint !== undefined && "output" in checkpoint ? checkpoint.output : undefined; } - getStageSession(workflowId: string, replayKey: string): { sessionId?: string; sessionFile?: string } | undefined { + getStageSession(workflowId: string, replayKey: string): { + sessionId?: string; + sessionFile?: string; + startedAt?: number; + durationMs?: number; + } | undefined { const checkpoint = this.workflows.get(workflowId)?.stageSessionByReplayKey.get(replayKey); if (checkpoint?.sessionId === undefined && checkpoint?.sessionFile === undefined) return undefined; - return { ...(checkpoint.sessionId !== undefined ? { sessionId: checkpoint.sessionId } : {}), ...(checkpoint.sessionFile !== undefined ? { sessionFile: checkpoint.sessionFile } : {}) }; + return { + ...(checkpoint.sessionId !== undefined ? { sessionId: checkpoint.sessionId } : {}), + ...(checkpoint.sessionFile !== undefined ? { sessionFile: checkpoint.sessionFile } : {}), + ...(checkpoint.startedAt !== undefined ? { startedAt: checkpoint.startedAt } : {}), + ...(checkpoint.durationMs !== undefined ? { durationMs: checkpoint.durationMs } : {}), + }; } listCheckpoints(workflowId: string): readonly DurableCheckpoint[] { diff --git a/packages/workflows/src/durable/scoped-backend.ts b/packages/workflows/src/durable/scoped-backend.ts index 3266c08a0..f731eda30 100644 --- a/packages/workflows/src/durable/scoped-backend.ts +++ b/packages/workflows/src/durable/scoped-backend.ts @@ -86,7 +86,12 @@ export class ScopedDurableBackend implements DurableWorkflowBackend { return this.inner.getStageOutput(this.scope.rootWorkflowId, this.scopeKey(replayKey)); } - getStageSession(_workflowId: string, replayKey: string): { sessionId?: string; sessionFile?: string } | undefined { + getStageSession(_workflowId: string, replayKey: string): { + sessionId?: string; + sessionFile?: string; + startedAt?: number; + durationMs?: number; + } | undefined { return this.inner.getStageSession(this.scope.rootWorkflowId, this.scopeKey(replayKey)); } diff --git a/packages/workflows/src/durable/stage-primitive.ts b/packages/workflows/src/durable/stage-primitive.ts index 1d20b1d59..fb32913c1 100644 --- a/packages/workflows/src/durable/stage-primitive.ts +++ b/packages/workflows/src/durable/stage-primitive.ts @@ -7,6 +7,7 @@ import type { DurableWorkflowBackend } from "./backend.js"; import type { ParallelFailFastScope } from "../runs/foreground/executor-types.js"; import { durableHash } from "./backend.js"; import { recordCheckpointDurably } from "./tool-primitive.js"; +import { elapsedStageMs } from "../shared/timing.js"; import { RESUME_CONTINUATION_PROMPT } from "../shared/resume-continuation.js"; import type { DurableStageCheckpoint } from "./types.js"; export type DurableCompletedStageCheckpoint = DurableStageCheckpoint & { readonly output: WorkflowSerializableValue }; @@ -17,6 +18,7 @@ export interface DurableStageDeps { readonly nextCheckpointId: () => string; readonly nextReplayKey: (stageName: string) => string; readonly replayKeyForCompletedStage?: (stage: StageSnapshot) => string | undefined; + readonly now?: () => number; } export async function recordStageCheckpoint(deps: DurableStageDeps, stage: StageSnapshot): Promise { @@ -51,17 +53,25 @@ export async function recordStageCheckpoint(deps: DurableStageDeps, stage: Stage export async function recordStageSessionCheckpoint(deps: DurableStageDeps, stage: StageSnapshot): Promise { const replayKey = deps.replayKeyForCompletedStage?.(stage) ?? stage.replayKey ?? deps.nextReplayKey(stage.name); if (stage.sessionFile === undefined) return false; + const checkpointNow = deps.now?.() ?? Date.now(); + const durationMs = elapsedStageMs(stage, checkpointNow) ?? 0; const current = deps.backend.getStageSession(deps.workflowId, replayKey); - if (current?.sessionFile === stage.sessionFile) return false; + if (current !== undefined + && current.sessionId === stage.sessionId + && current.sessionFile === stage.sessionFile + && current.startedAt === stage.startedAt + && current.durationMs === durationMs) return false; const checkpoint: DurableStageCheckpoint = { kind: "stage", workflowId: deps.workflowId, - checkpointId: stageSessionCheckpointId(replayKey, stage), + checkpointId: stageSessionCheckpointId(replayKey, stage, durationMs), name: stage.name, replayKey, ...(stage.sessionId !== undefined ? { sessionId: stage.sessionId } : {}), sessionFile: stage.sessionFile, - completedAt: Date.now(), + ...(stage.startedAt !== undefined ? { startedAt: stage.startedAt } : {}), + durationMs, + completedAt: checkpointNow, }; await recordCheckpointDurably(deps.backend, checkpoint); return true; @@ -107,7 +117,10 @@ export function createDurableStagePrimitive(input: { const liveOptions: StageOptions | undefined = { ...(options ?? {}), durableReplayKey: replayKey, - ...(isMidSessionResume ? { resumeFromSessionFile: session.sessionFile } : {}), + ...(isMidSessionResume ? { + resumeFromSessionFile: session.sessionFile, + durableAccumulatedDurationMs: session.durationMs ?? 0, + } : {}), }; const live = withMidSessionResumePrompt(input.stage(name, liveOptions, replayKey), isMidSessionResume); if (options?.schema === undefined) return live; @@ -139,7 +152,10 @@ export function createDurableTaskPrimitive(input: { const taskOptions: WorkflowTaskOptions = { ...options, durableReplayKey: replayKey, - ...(session?.sessionFile !== undefined ? { resumeFromSessionFile: session.sessionFile } : {}), + ...(session?.sessionFile !== undefined ? { + resumeFromSessionFile: session.sessionFile, + durableAccumulatedDurationMs: session.durationMs ?? 0, + } : {}), }; const result = await input.task(name, taskOptions, stageFailFastScope); await recordCheckpointDurably(input.backend, { @@ -317,10 +333,12 @@ export function stableCheckpointId(kind: string, replayKey: string): string { return `${kind}:${replayKey}`; } -function stageSessionCheckpointId(replayKey: string, stage: StageSnapshot): string { +function stageSessionCheckpointId(replayKey: string, stage: StageSnapshot, durationMs: number): string { return `${stableCheckpointId("stage-session", replayKey)}:${durableHash({ sessionId: stage.sessionId ?? "", sessionFile: stage.sessionFile ?? "", + startedAt: stage.startedAt ?? 0, + durationMs, })}`; } diff --git a/packages/workflows/src/runs/foreground/executor-stage-call.ts b/packages/workflows/src/runs/foreground/executor-stage-call.ts index 1885e577f..28ecfcfad 100644 --- a/packages/workflows/src/runs/foreground/executor-stage-call.ts +++ b/packages/workflows/src/runs/foreground/executor-stage-call.ts @@ -7,6 +7,7 @@ import { raceAbort } from "./executor-abort.js"; import { hasExplicitFastModeCandidate } from "./executor-direct-helpers.js"; import { applyFailureToStage } from "./executor-lifecycle.js"; import { isTerminalStage } from "./executor-scheduler.js"; +import { rebasedStageStartedAt } from "../../shared/timing.js"; export interface TrackedStageCallOptions { readonly eagerSession?: boolean; @@ -110,8 +111,9 @@ export function createTrackedStageCaller(input: { } } if (trackStageLifecycle) { + const now = Date.now(); runtime.stageSnapshot.status = "running"; - runtime.stageSnapshot.startedAt = Date.now(); + runtime.stageSnapshot.startedAt = rebasedStageStartedAt(input.options?.durableAccumulatedDurationMs, now); const hasNoExplicitModelConfig = input.options?.model === undefined && input.options?.fallbackModels === undefined; const promptAdapterHandlesInitialPrompt = input.adapters.prompt !== undefined; if (callOptions.eagerSession && !promptAdapterHandlesInitialPrompt && (hasNoExplicitModelConfig || await hasExplicitFastModeCandidate({ diff --git a/packages/workflows/src/runs/foreground/stage-runner-options.ts b/packages/workflows/src/runs/foreground/stage-runner-options.ts index 1a8adad05..db677e8bf 100644 --- a/packages/workflows/src/runs/foreground/stage-runner-options.ts +++ b/packages/workflows/src/runs/foreground/stage-runner-options.ts @@ -20,6 +20,7 @@ export function stripWorkflowOnlyOptions( forkFromSessionFile, resumeFromSessionFile, durableReplayKey: _durableReplayKey, + durableAccumulatedDurationMs: _durableAccumulatedDurationMs, sessionDir, gitWorktreeDir: _gitWorktreeDir, baseBranch: _baseBranch, diff --git a/packages/workflows/src/shared/timing.ts b/packages/workflows/src/shared/timing.ts index 73f8a3279..c51fdbc3a 100644 --- a/packages/workflows/src/shared/timing.ts +++ b/packages/workflows/src/shared/timing.ts @@ -18,6 +18,10 @@ function nonNegative(ms: number): number { return Math.max(0, ms); } +export function rebasedStageStartedAt(accumulatedDurationMs: number | undefined, resumedAt: number): number { + return resumedAt - nonNegative(accumulatedDurationMs ?? 0); +} + function elapsedFromStart( startedAt: number, now: number, diff --git a/packages/workflows/src/shared/types.ts b/packages/workflows/src/shared/types.ts index 3f34c06e6..f66e4dca4 100644 --- a/packages/workflows/src/shared/types.ts +++ b/packages/workflows/src/shared/types.ts @@ -191,6 +191,8 @@ export interface StageOptions { assert.equal(backend2.getWorkflow(WORKFLOW_ID)!.name, "file-workflow"); }); + test("preserves latest active-stage timing across a file process boundary", () => { + const replayKey = "stage:analyze:1"; + backend.recordCheckpoint({ + kind: "stage", workflowId: WORKFLOW_ID, checkpointId: "stage-session:1", name: "analyze", replayKey, + sessionFile: "/tmp/analyze.jsonl", startedAt: 1000, durationMs: 400, completedAt: 1400, + }); + backend.recordCheckpoint({ + kind: "stage", workflowId: WORKFLOW_ID, checkpointId: "stage-session:2", name: "analyze", replayKey, + sessionFile: "/tmp/analyze.jsonl", startedAt: 1000, durationMs: 750, completedAt: 1750, + }); + + const fresh = new FileDurableBackend(join(tmpDir, "state.json")); + assert.deepEqual(fresh.getStageSession(WORKFLOW_ID, replayKey), { + sessionFile: "/tmp/analyze.jsonl", + startedAt: 1000, + durationMs: 750, + }); + }); + test("lists resumable workflows from a new backend instance", () => { backend.recordCheckpoint(makeToolCheckpoint(WORKFLOW_ID, "progress", "h-progress", "done")); backend.setWorkflowStatus(WORKFLOW_ID, "paused"); diff --git a/test/unit/durable-dbos-backend.test.ts b/test/unit/durable-dbos-backend.test.ts index 75c92774f..4794ff66c 100644 --- a/test/unit/durable-dbos-backend.test.ts +++ b/test/unit/durable-dbos-backend.test.ts @@ -300,15 +300,22 @@ describe("DbosDurableBackend hydration (fresh process)", () => { test("hydrateWorkflow reconstructs stage checkpoints from DBOS envelopes", async () => { const cp: DurableStageCheckpoint = { - kind: "stage", workflowId: "wf-h3", checkpointId: "stage:r1", name: "build", replayKey: "stage:build:1", output: "done", completedAt: 3000, + kind: "stage", workflowId: "wf-h3", checkpointId: "stage:r1", name: "build", replayKey: "stage:build:1", output: "done", completedAt: 1400, + sessionFile: "/tmp/build.jsonl", startedAt: 1000, durationMs: 400, }; seedMockWorkflow(sdk, { workflowId: "wf-h3", name: "test", status: "PENDING" }); seedMockCheckpoint(sdk, "wf-h3", cp); + seedMockCheckpoint(sdk, "wf-h3", { + ...cp, checkpointId: "stage-session:r2", output: undefined, durationMs: 900, completedAt: 1900, + }); const fresh = new DbosDurableBackend(sdk); assert.equal(fresh.getStageOutput("wf-h3", "stage:build:1"), undefined); await fresh.hydrateWorkflow("wf-h3"); + await fresh.hydrateWorkflow("wf-h3"); assert.equal(fresh.getStageOutput("wf-h3", "stage:build:1"), "done"); + assert.equal(fresh.getStageSession("wf-h3", "stage:build:1")?.durationMs, 900); + assert.equal(fresh.listCheckpoints("wf-h3").length, 2); }); test("hydrateWorkflow handles legacy/simple payloads gracefully", async () => { diff --git a/test/unit/durable-stage-session-resume.test.ts b/test/unit/durable-stage-session-resume.test.ts index 2267c5167..b52b38f23 100644 --- a/test/unit/durable-stage-session-resume.test.ts +++ b/test/unit/durable-stage-session-resume.test.ts @@ -1,11 +1,18 @@ -import { beforeEach, describe, test } from "bun:test"; +import { afterEach, beforeEach, describe, mock, spyOn, test } from "bun:test"; import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { InMemoryDurableBackend } from "../../packages/workflows/src/durable/backend.js"; +import { FileDurableBackend } from "../../packages/workflows/src/durable/file-backend.js"; import { createCheckpointIdGenerator } from "../../packages/workflows/src/durable/tool-primitive.js"; import { createDurableStagePrimitive, createStageReplayKeyGenerator, recordStageCheckpoint, recordStageSessionCheckpoint } from "../../packages/workflows/src/durable/stage-primitive.js"; import { RESUME_CONTINUATION_PROMPT } from "../../packages/workflows/src/runs/foreground/executor.js"; import type { StageSnapshot } from "../../packages/workflows/src/shared/store-types.js"; +import { elapsedStageMs, rebasedStageStartedAt } from "../../packages/workflows/src/shared/timing.js"; +import { createStore, run, Type, workflow } from "./executor-shared.js"; +afterEach(() => mock.restore()); const WORKFLOW_ID = "wf-stage-session-resume"; function makeStage(overrides: Partial = {}): StageSnapshot { @@ -59,12 +66,13 @@ describe("durable stage session resume", () => { }); }); - function deps() { + function deps(now = 2000) { return { workflowId: WORKFLOW_ID, backend, nextCheckpointId: createCheckpointIdGenerator(), nextReplayKey: createStageReplayKeyGenerator(WORKFLOW_ID), + now: () => now, }; } @@ -75,6 +83,8 @@ describe("durable stage session resume", () => { assert.deepEqual(backend.getStageSession(WORKFLOW_ID, "stage:analyze:1"), { sessionId: "sid-1", sessionFile: "/tmp/stage.jsonl", + startedAt: 1000, + durationMs: 1000, }); // Running (active) workflows are hidden from resume; quitting flips the // durable handle to paused, which is when an in-progress stage session @@ -83,6 +93,47 @@ describe("durable stage session resume", () => { assert.equal(backend.listResumableWorkflows().length, 1); }); + test("refreshes accumulated active duration for repeated checkpoints of one session", async () => { + const replayKey = "stage:analyze:1"; + const stage = makeStage({ replayKey, sessionId: "sid-1", sessionFile: "/tmp/stage.jsonl" }); + + assert.equal(await recordStageSessionCheckpoint(deps(1400), stage), true); + assert.equal(await recordStageSessionCheckpoint(deps(1750), stage), true); + assert.equal(await recordStageSessionCheckpoint(deps(1750), stage), false); + + assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { + sessionId: "sid-1", + sessionFile: "/tmp/stage.jsonl", + startedAt: 1000, + durationMs: 750, + }); + assert.equal(backend.listCheckpoints(WORKFLOW_ID).length, 2); + }); + + test("checkpoints pause-adjusted duration without double-counting", async () => { + const replayKey = "stage:analyze:1"; + const stage = makeStage({ + replayKey, + sessionFile: "/tmp/stage.jsonl", + pausedDurationMs: 200, + pausedAt: 1800, + }); + + await recordStageSessionCheckpoint(deps(2200), stage); + + assert.equal(backend.getStageSession(WORKFLOW_ID, replayKey)?.durationMs, 600); + }); + + test("counts post-resume elapsed time while excluding a new pause exactly once", () => { + const resumedAt = 5000; + const startedAt = rebasedStageStartedAt(700, resumedAt); + const completedAt = 5500; + + assert.equal(startedAt, 4300); + assert.equal(elapsedStageMs({ startedAt, pausedDurationMs: 200 }, completedAt), 1000); + assert.equal(rebasedStageStartedAt(-50, resumedAt), resumedAt); + }); + test("reopens prior session file when output is not completed", async () => { const replayKey = "stage:analyze:1"; await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/prior.jsonl" })); @@ -108,6 +159,24 @@ describe("durable stage session resume", () => { assert.equal(observedPrompt, RESUME_CONTINUATION_PROMPT); }); + test("hydrates accumulated duration into a new-process live stage", async () => { + const replayKey = "stage:analyze:1"; + await recordStageSessionCheckpoint(deps(1700), makeStage({ replayKey, sessionFile: "/tmp/prior.jsonl" })); + let accumulatedDurationMs: number | undefined; + const stage = createDurableStagePrimitive({ + workflowId: WORKFLOW_ID, + backend, + nextReplayKey: () => replayKey, + stage: (_name, options) => { + accumulatedDurationMs = options?.durableAccumulatedDurationMs; + return fakeStageContext("resumed"); + }, + }); + + await stage("analyze").prompt("continue"); + assert.equal(accumulatedDurationMs, 700); + }); + test("mid-session resume does not eagerly read throwing StageContext getters", async () => { const replayKey = "stage:analyze:1"; await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/prior.jsonl" })); @@ -141,9 +210,13 @@ describe("durable stage session resume", () => { test("updates session metadata across repeated resumes", async () => { const replayKey = "stage:analyze:1"; - assert.equal(await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/first.jsonl" })), true); - assert.equal(await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/second.jsonl" })), true); - assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/second.jsonl" }); + assert.equal(await recordStageSessionCheckpoint(deps(1500), makeStage({ replayKey, sessionFile: "/tmp/first.jsonl" })), true); + assert.equal(await recordStageSessionCheckpoint(deps(1800), makeStage({ replayKey, sessionFile: "/tmp/second.jsonl" })), true); + assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { + sessionFile: "/tmp/second.jsonl", + startedAt: 1000, + durationMs: 800, + }); }); test("completed output wins over later session metadata", async () => { @@ -151,7 +224,7 @@ describe("durable stage session resume", () => { await recordStageCheckpoint(deps(), makeStage({ status: "completed", replayKey, result: "done", endedAt: 2000 })); await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/later.jsonl" })); assert.equal(backend.getStageOutput(WORKFLOW_ID, replayKey), "done"); - assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/later.jsonl" }); + assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/later.jsonl", startedAt: 1000, durationMs: 1000 }); const stage = createDurableStagePrimitive({ workflowId: WORKFLOW_ID, backend, @@ -166,6 +239,73 @@ describe("durable stage session resume", () => { await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/first.jsonl" })); await recordStageCheckpoint(deps(), makeStage({ status: "completed", replayKey, result: "done", endedAt: 2000 })); assert.equal(backend.getStageOutput(WORKFLOW_ID, replayKey), "done"); - assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/first.jsonl" }); + assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/first.jsonl", startedAt: 1000, durationMs: 1000 }); + }); + + test("file process-boundary completion preserves total duration and replay identity", async () => { + const dir = mkdtempSync(join(tmpdir(), "atomic-stage-duration-")); + try { + const runId = "wf-stage-duration-resume"; + const replayKey = "stage:analyze:1"; + const stateFile = join(dir, "durable.json"); + const writer = new FileDurableBackend(stateFile); + writer.registerWorkflow({ workflowId: runId, name: "duration-resume", inputs: {}, createdAt: 1000, status: "paused" }); + await recordStageSessionCheckpoint({ + workflowId: runId, + backend: writer, + nextCheckpointId: createCheckpointIdGenerator(), + nextReplayKey: () => replayKey, + now: () => 1700, + }, makeStage({ replayKey, sessionFile: "/tmp/durable-stage-duration.jsonl" })); + + let clock = 5000; + let liveStageCalls = 0; + spyOn(Date, "now").mockImplementation(() => clock); + let lifecycleDurationMs: number | undefined; + const store = createStore(); + const def = workflow({ + name: "duration-resume", + description: "", + inputs: {}, + outputs: { result: Type.String() }, + run: async (ctx) => ({ result: await ctx.stage("analyze").complete("done") }), + }); + const resumedBackend = new FileDurableBackend(stateFile); + const first = await run(def, {}, { + runId, + store, + durableBackend: resumedBackend, + adapters: { complete: { complete: async (text) => { + liveStageCalls += 1; + clock = 5300; + return text; + } } }, + onStageEnd: (_stageRunId, snapshot) => { lifecycleDurationMs = snapshot.durationMs; }, + }); + + const storedStage = store.runs()[0]?.stages.find((stage) => stage.name === "analyze"); + const durableStage = resumedBackend.listCheckpoints(runId).find((checkpoint) => + checkpoint.kind === "stage" && checkpoint.replayKey === replayKey && checkpoint.output !== undefined, + ); + assert.equal(first.status, "completed"); + assert.equal(liveStageCalls, 1); + assert.equal(storedStage?.durationMs, 1000); + assert.equal(lifecycleDurationMs, 1000); + assert.equal(durableStage?.kind === "stage" ? durableStage.durationMs : undefined, 1000); + + const replay = await run(def, {}, { + runId, + store: createStore(), + durableBackend: new FileDurableBackend(stateFile), + adapters: { complete: { complete: async () => { + liveStageCalls += 1; + throw new Error("completed stage replay must not execute again"); + } } }, + }); + assert.equal(replay.status, "completed"); + assert.equal(liveStageCalls, 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); }); From 128b2175cf1809e000b57e860f4745822c0989db Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Sun, 12 Jul 2026 02:13:18 -0700 Subject: [PATCH 2/3] fix(workflows): preserve resumed duration across tracked calls Apply the durable timing baseline only once per live stage and cover concurrent tracked calls across process-boundary resume and replay. Co-Authored-By: GPT-5.6 Sol --- packages/workflows/CHANGELOG.md | 8 +++--- .../runs/foreground/executor-stage-call.ts | 2 +- .../unit/durable-stage-session-resume.test.ts | 25 +++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 9e42ceb0c..10a95972e 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ## [Unreleased] +### Fixed + +- Fixed durable mid-stage workflow resume to refresh pause-adjusted active-stage duration at repeated session checkpoints, preserve it through file and DBOS hydration, and continue accumulating it after a process-boundary resume without resetting the baseline when concurrent tracked stage calls begin, changing replay identity, or re-running completed side effects ([#1713](https://github.com/bastani-inc/atomic/issues/1713)). + ## [0.9.8] - 2026-07-12 ### Changed @@ -86,10 +90,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Restored Ralph's builtin implementation-stage prompts to require subagent-led investigation, editing, and validation, reversing the selective direct-implementation wording introduced with intent-first routing. - Expanded model-facing workflow guidance to treat composition as a first-class design option: custom parents can import reusable project/package workflows or bundled builtin definitions, invoke them through `ctx.workflow(...)`, and nest further child workflows within `maxDepth` while preserving expanded graph visibility, HIL, durability, controls, and declared output contracts. -### Fixed - -- Fixed durable mid-stage workflow resume to refresh pause-adjusted active-stage duration at repeated session checkpoints, preserve it through file and DBOS hydration, and continue accumulating it after a process-boundary resume without changing replay identity or re-running completed side effects ([#1713](https://github.com/bastani-inc/atomic/issues/1713)). - ## [0.9.5] - 2026-07-11 ### Breaking Changes diff --git a/packages/workflows/src/runs/foreground/executor-stage-call.ts b/packages/workflows/src/runs/foreground/executor-stage-call.ts index 28ecfcfad..5ee32f4a4 100644 --- a/packages/workflows/src/runs/foreground/executor-stage-call.ts +++ b/packages/workflows/src/runs/foreground/executor-stage-call.ts @@ -113,7 +113,7 @@ export function createTrackedStageCaller(input: { if (trackStageLifecycle) { const now = Date.now(); runtime.stageSnapshot.status = "running"; - runtime.stageSnapshot.startedAt = rebasedStageStartedAt(input.options?.durableAccumulatedDurationMs, now); + runtime.stageSnapshot.startedAt ??= rebasedStageStartedAt(input.options?.durableAccumulatedDurationMs, now); const hasNoExplicitModelConfig = input.options?.model === undefined && input.options?.fallbackModels === undefined; const promptAdapterHandlesInitialPrompt = input.adapters.prompt !== undefined; if (callOptions.eagerSession && !promptAdapterHandlesInitialPrompt && (hasNoExplicitModelConfig || await hasExplicitFastModeCandidate({ diff --git a/test/unit/durable-stage-session-resume.test.ts b/test/unit/durable-stage-session-resume.test.ts index b52b38f23..262e6637e 100644 --- a/test/unit/durable-stage-session-resume.test.ts +++ b/test/unit/durable-stage-session-resume.test.ts @@ -242,7 +242,7 @@ describe("durable stage session resume", () => { assert.deepEqual(backend.getStageSession(WORKFLOW_ID, replayKey), { sessionFile: "/tmp/first.jsonl", startedAt: 1000, durationMs: 1000 }); }); - test("file process-boundary completion preserves total duration and replay identity", async () => { + test("file process-boundary completion preserves duration across concurrent tracked calls and replay", async () => { const dir = mkdtempSync(join(tmpdir(), "atomic-stage-duration-")); try { const runId = "wf-stage-duration-resume"; @@ -260,6 +260,8 @@ describe("durable stage session resume", () => { let clock = 5000; let liveStageCalls = 0; + let releaseCalls: () => void = () => {}; + const bothCallsStarted = new Promise((resolve) => { releaseCalls = resolve; }); spyOn(Date, "now").mockImplementation(() => clock); let lifecycleDurationMs: number | undefined; const store = createStore(); @@ -268,7 +270,14 @@ describe("durable stage session resume", () => { description: "", inputs: {}, outputs: { result: Type.String() }, - run: async (ctx) => ({ result: await ctx.stage("analyze").complete("done") }), + run: async (ctx) => { + const stage = ctx.stage("analyze"); + await Promise.allSettled([ + stage.complete("first"), + stage.complete("second"), + ]); + return { result: "done" }; + }, }); const resumedBackend = new FileDurableBackend(stateFile); const first = await run(def, {}, { @@ -277,7 +286,13 @@ describe("durable stage session resume", () => { durableBackend: resumedBackend, adapters: { complete: { complete: async (text) => { liveStageCalls += 1; - clock = 5300; + if (liveStageCalls === 1) { + clock = 5100; + await bothCallsStarted; + } else { + clock = 5300; + releaseCalls(); + } return text; } } }, onStageEnd: (_stageRunId, snapshot) => { lifecycleDurationMs = snapshot.durationMs; }, @@ -288,7 +303,7 @@ describe("durable stage session resume", () => { checkpoint.kind === "stage" && checkpoint.replayKey === replayKey && checkpoint.output !== undefined, ); assert.equal(first.status, "completed"); - assert.equal(liveStageCalls, 1); + assert.equal(liveStageCalls, 2); assert.equal(storedStage?.durationMs, 1000); assert.equal(lifecycleDurationMs, 1000); assert.equal(durableStage?.kind === "stage" ? durableStage.durationMs : undefined, 1000); @@ -303,7 +318,7 @@ describe("durable stage session resume", () => { } } }, }); assert.equal(replay.status, "completed"); - assert.equal(liveStageCalls, 1); + assert.equal(liveStageCalls, 2); } finally { rmSync(dir, { recursive: true, force: true }); } From 2d6ed612ec9c588b66413f73def11e50a56d2373 Mon Sep 17 00:00:00 2001 From: Norin Lavaee Date: Sun, 12 Jul 2026 21:02:57 -0700 Subject: [PATCH 3/3] test(workflows): cover repeated durable duration resume Assistant-model: GPT-5.6 Sol --- packages/coding-agent/docs/workflows.md | 2 +- packages/workflows/CHANGELOG.md | 2 +- packages/workflows/README.md | 2 +- .../unit/durable-stage-session-resume.test.ts | 94 ++++++++++++++++++- 4 files changed, 96 insertions(+), 4 deletions(-) diff --git a/packages/coding-agent/docs/workflows.md b/packages/coding-agent/docs/workflows.md index ca1a04591..2b7281e23 100644 --- a/packages/coding-agent/docs/workflows.md +++ b/packages/coding-agent/docs/workflows.md @@ -410,7 +410,7 @@ Named runs go to the background. Common controls: When a paused stage is resumed with a message, Atomic lets the stage answer that resume message, then (if the stage has not already finalized) injects `Continue where you left off. If you believe you are finished with your original task (or a redefined task if the user told you), stop.` into the same stage session before normal stage completion/readiness handling. This keeps interrupted work moving without asking you to manually type a second continuation prompt while also preventing stages that already finished their scoped work from overstepping. -Durable `/workflow resume` preserves completed stage metadata, active-stage elapsed time, and graph topology. While an LM stage is active, repeated durable checkpoints refresh its accumulated pause-adjusted duration even when its session file does not change. When a new Atomic process reopens that unfinished session mid-chat, timing continues from the saved baseline and uses the same continuation prompt shown above, so status, graph, stored, and lifecycle duration includes both pre-resume and post-resume work without double-counting pauses. Replayed `ctx.stage`, `ctx.task`, `ctx.chain`, `ctx.parallel`, and child-workflow checkpoints keep their original summaries, timing, session/model metadata, and parallel fanout parentage instead of appearing as freshly flattened replay nodes. +Durable `/workflow resume` preserves completed stage metadata, active-stage elapsed time, and graph topology. While an LM stage or task is active, repeated durable checkpoints refresh its accumulated pause-adjusted duration even when its session file does not change. Each new Atomic process that reopens the unfinished session mid-chat starts from the latest saved baseline and uses the same continuation prompt shown above, so repeated process-boundary resumes keep status, graph, stored, and lifecycle duration cumulative without double-counting pauses from earlier process segments. Replayed `ctx.stage`, `ctx.task`, `ctx.chain`, `ctx.parallel`, and child-workflow checkpoints keep their original summaries, timing, session/model metadata, and parallel fanout parentage instead of appearing as freshly flattened replay nodes. Workflow stage sessions are marked as **internal** and excluded from the standard `/resume`, `atomic -r`, and `--continue` history so they do not clutter your interactive session picker. They remain resumable and inspectable through the workflow-specific commands and tool actions shown here (`/workflow resume`, `/workflow attach`, `workflow({ action: "status" | "stages" | "stage" | "resume" })`), which read the run/stage store and its `sessionFile` links directly. Passing a stage session's file path to `--session` still opens it explicitly. Legacy workflow sessions created before this marker behavior lack the signal and will continue to appear in the standard history until they age out. diff --git a/packages/workflows/CHANGELOG.md b/packages/workflows/CHANGELOG.md index 10a95972e..80dd10c08 100644 --- a/packages/workflows/CHANGELOG.md +++ b/packages/workflows/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed -- Fixed durable mid-stage workflow resume to refresh pause-adjusted active-stage duration at repeated session checkpoints, preserve it through file and DBOS hydration, and continue accumulating it after a process-boundary resume without resetting the baseline when concurrent tracked stage calls begin, changing replay identity, or re-running completed side effects ([#1713](https://github.com/bastani-inc/atomic/issues/1713)). +- Fixed durable mid-stage workflow resume to refresh pause-adjusted active stage/task duration at repeated session checkpoints, preserve it through file and DBOS hydration, and continue accumulating it across repeated process-boundary resumes without resetting the baseline when concurrent tracked stage calls begin, double-counting earlier pauses, changing replay identity, or re-running completed side effects ([#1713](https://github.com/bastani-inc/atomic/issues/1713)). ## [0.9.8] - 2026-07-12 diff --git a/packages/workflows/README.md b/packages/workflows/README.md index f4310e386..8b6fb81fb 100644 --- a/packages/workflows/README.md +++ b/packages/workflows/README.md @@ -544,7 +544,7 @@ Input overrides are bare `key=value` tokens (no leading `--`). Values are JSON-p Named workflow launches always run as **background tasks** in interactive sessions. Model-launched direct `task`, `tasks`, and `chain` calls must set top-level `async: true` so the chat editor stays free; inspection and control calls are unaffected. Foreground launches are reserved for explicit user requests or technical requirements, with notice before launch. Press **F2** (or `/workflow connect `) to attach to the live graph viewer; HIL prompts (`ctx.ui.input/confirm/select/editor/custom`) appear as awaiting-input graph nodes. Press Enter on a focused node, or click a visible graph node directly, to open that stage and answer locally, never as a modal dialog over the chat. Attached stage chats capture mouse/trackpad wheel events by default so scrolling stays inside the active stage transcript or prompt instead of falling through to terminal/main-chat scrollback. Press `ctrl+t` to toggle **copy mode**: copy mode disables workflow-chat mouse reporting so normal terminal/tmux text selection can work; press `ctrl+t` again to leave copy mode and restore workflow-chat scrolling. Archived read-only stage transcripts show the same copy-mode footer/status, allowing their transcript text to be selected and copied while preserving `esc` close and `ctrl+d` graph navigation. While copy mode is on, wheel/trackpad gestures are handled by the terminal/tmux and may scroll terminal scrollback, so leave copy mode before using the wheel again. Human input is detected when those runtime `ctx.ui.*` calls execute; workflows no longer have a declaration-time HIL flag. -Durable resume preserves both completed and active stage timing. Replayed stage/task/parallel/child checkpoints retain summaries, durations, session/model metadata, and parallel branch parentage in the graph instead of flattening fanout branches into a sequential replay chain. While an LM stage is active, repeated durable session checkpoints refresh its pause-adjusted accumulated duration even when the session file is unchanged; a later `/workflow resume` continues from that baseline, so status, graph, stored, and lifecycle durations include the elapsed segments from before and after the process boundary without counting paused time twice. +Durable resume preserves both completed and active stage timing. Replayed stage/task/parallel/child checkpoints retain summaries, durations, session/model metadata, and parallel branch parentage in the graph instead of flattening fanout branches into a sequential replay chain. While an LM stage or task is active, repeated durable session checkpoints refresh its pause-adjusted accumulated duration even when the session file is unchanged. Every later `/workflow resume` starts from the latest saved baseline, so repeated process-boundary resumes keep status, graph, stored, and lifecycle durations cumulative while subtracting only pauses from the current process segment. Nested `ctx.workflow(...)` calls are displayed as an expanded graph within the top-level run. `/workflow status` and run pickers list only top-level user-launched workflows, not implementation-owned child runs. The `workflow` tool's `stages`, `stage`, `transcript`, `send`, `pause`, `interrupt`, and `resume` actions can still target visible child stage ids, prefixes, or names from the expanded graph; Atomic routes the control action to the owning nested run internally. (`stages`, `stage`, `transcript`, and `send` are `workflow` tool actions, not `/workflow` slash subcommands; the slash command exposes `connect`, `attach`, `pause`, `list`, `status`, `interrupt`, `kill`, `resume`, `reload`, and `inputs`.) diff --git a/test/unit/durable-stage-session-resume.test.ts b/test/unit/durable-stage-session-resume.test.ts index 262e6637e..fc86117e4 100644 --- a/test/unit/durable-stage-session-resume.test.ts +++ b/test/unit/durable-stage-session-resume.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { InMemoryDurableBackend } from "../../packages/workflows/src/durable/backend.js"; import { FileDurableBackend } from "../../packages/workflows/src/durable/file-backend.js"; import { createCheckpointIdGenerator } from "../../packages/workflows/src/durable/tool-primitive.js"; -import { createDurableStagePrimitive, createStageReplayKeyGenerator, recordStageCheckpoint, recordStageSessionCheckpoint } from "../../packages/workflows/src/durable/stage-primitive.js"; +import { createDurableStagePrimitive, createDurableTaskPrimitive, createStageReplayKeyGenerator, recordStageCheckpoint, recordStageSessionCheckpoint } from "../../packages/workflows/src/durable/stage-primitive.js"; import { RESUME_CONTINUATION_PROMPT } from "../../packages/workflows/src/runs/foreground/executor.js"; import type { StageSnapshot } from "../../packages/workflows/src/shared/store-types.js"; import { elapsedStageMs, rebasedStageStartedAt } from "../../packages/workflows/src/shared/timing.js"; @@ -177,6 +177,25 @@ describe("durable stage session resume", () => { assert.equal(accumulatedDurationMs, 700); }); + test("hydrates accumulated duration into a resumed task", async () => { + const replayKey = "stage:task:analyze:1"; + await recordStageSessionCheckpoint(deps(1700), makeStage({ replayKey, sessionFile: "/tmp/prior-task.jsonl" })); + let observedOptions: { resumeFromSessionFile?: string; durableAccumulatedDurationMs?: number } | undefined; + const task = createDurableTaskPrimitive({ + workflowId: WORKFLOW_ID, + backend, + nextReplayKey: () => replayKey, + task: async (_name, options) => { + observedOptions = options; + return { name: "analyze", stageName: "analyze", text: "resumed task" }; + }, + }); + + assert.equal((await task("analyze", { prompt: "continue" })).text, "resumed task"); + assert.equal(observedOptions?.resumeFromSessionFile, "/tmp/prior-task.jsonl"); + assert.equal(observedOptions?.durableAccumulatedDurationMs, 700); + }); + test("mid-session resume does not eagerly read throwing StageContext getters", async () => { const replayKey = "stage:analyze:1"; await recordStageSessionCheckpoint(deps(), makeStage({ replayKey, sessionFile: "/tmp/prior.jsonl" })); @@ -219,6 +238,79 @@ describe("durable stage session resume", () => { }); }); + test("preserves pause-adjusted duration across two process-boundary resumes", async () => { + const dir = mkdtempSync(join(tmpdir(), "atomic-stage-repeated-resume-")); + try { + const runId = "wf-stage-repeated-resume"; + const replayKey = "stage:analyze:1"; + const stateFile = join(dir, "durable.json"); + const firstProcess = new FileDurableBackend(stateFile); + firstProcess.registerWorkflow({ workflowId: runId, name: "repeated-resume", inputs: {}, createdAt: 1000, status: "paused" }); + await recordStageSessionCheckpoint({ + workflowId: runId, + backend: firstProcess, + nextCheckpointId: createCheckpointIdGenerator(), + nextReplayKey: () => replayKey, + now: () => 1800, + }, makeStage({ replayKey, sessionFile: "/tmp/process-a.jsonl", pausedDurationMs: 100 })); + + const secondProcess = new FileDurableBackend(stateFile); + const secondBaseline = secondProcess.getStageSession(runId, replayKey)?.durationMs; + assert.equal(secondBaseline, 700); + await recordStageSessionCheckpoint({ + workflowId: runId, + backend: secondProcess, + nextCheckpointId: createCheckpointIdGenerator(), + nextReplayKey: () => replayKey, + now: () => 5600, + }, makeStage({ + replayKey, + sessionFile: "/tmp/process-b.jsonl", + startedAt: rebasedStageStartedAt(secondBaseline, 5000), + pausedDurationMs: 200, + })); + + const thirdProcess = new FileDurableBackend(stateFile); + const thirdBaseline = thirdProcess.getStageSession(runId, replayKey)?.durationMs; + assert.equal(thirdBaseline, 1100); + const completed = makeStage({ + replayKey, + sessionFile: "/tmp/process-c.jsonl", + status: "completed", + result: "done", + startedAt: rebasedStageStartedAt(thirdBaseline, 9000), + endedAt: 9500, + pausedDurationMs: 300, + durationMs: 1300, + }); + await recordStageSessionCheckpoint({ + workflowId: runId, + backend: thirdProcess, + nextCheckpointId: createCheckpointIdGenerator(), + nextReplayKey: () => replayKey, + now: () => 9500, + }, completed); + await recordStageCheckpoint({ + workflowId: runId, + backend: thirdProcess, + nextCheckpointId: createCheckpointIdGenerator(), + nextReplayKey: () => replayKey, + }, completed); + + const replayBackend = new FileDurableBackend(stateFile); + assert.equal(replayBackend.getStageSession(runId, replayKey)?.durationMs, 1300); + const replayed = createDurableStagePrimitive({ + workflowId: runId, + backend: replayBackend, + nextReplayKey: () => replayKey, + stage: () => { throw new Error("completed stage must not run after repeated resume"); }, + }); + assert.equal(await replayed("analyze").prompt("ignored"), "done"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("completed output wins over later session metadata", async () => { const replayKey = "stage:analyze:1"; await recordStageCheckpoint(deps(), makeStage({ status: "completed", replayKey, result: "done", endedAt: 2000 }));