diff --git a/packages/workflows/src/extension/index.ts b/packages/workflows/src/extension/index.ts index ce225398a..8323fc915 100644 --- a/packages/workflows/src/extension/index.ts +++ b/packages/workflows/src/extension/index.ts @@ -1795,7 +1795,11 @@ export function makeExecuteWorkflowTool( const isPaused = run?.status === "paused" || (run?.stages.some((s) => s.status === "paused") ?? false); - if (!isPaused && run?.status === "failed" && run.endedAt !== undefined && run.resumable !== false) { + const isResumableContinuation = run !== undefined && !isPaused && ( + (run.status === "failed" && run.endedAt !== undefined && run.resumable !== false) || + (run.endedAt === undefined && run.resumable === true && run.failureRecoverability === "recoverable") + ); + if (isResumableContinuation) { const continuation = activeRuntime.resumeFailedRun(stageRunId, stage.stageId, { policy }); return { action: "resume", @@ -3133,7 +3137,11 @@ function factory(pi: ExtensionAPI): void { const isPaused = run?.status === "paused" || (run?.stages.some((s) => s.status === "paused") ?? false); - if (!isPaused && run?.status === "failed" && run.endedAt !== undefined && run.resumable !== false) { + const isResumableContinuation = run !== undefined && !isPaused && ( + (run.status === "failed" && run.endedAt !== undefined && run.resumable !== false) || + (run.endedAt === undefined && run.resumable === true && run.failureRecoverability === "recoverable") + ); + if (isResumableContinuation) { const continuation = runtimeForContext(ctx).resumeFailedRun(stageRunId, stageId, { policy }); if (continuation.ok) { print(continuation.message); diff --git a/packages/workflows/src/extension/runtime.ts b/packages/workflows/src/extension/runtime.ts index eb56e3835..a37113562 100644 --- a/packages/workflows/src/extension/runtime.ts +++ b/packages/workflows/src/extension/runtime.ts @@ -44,6 +44,7 @@ import { import { validateWorkflowModels } from "../runs/shared/model-fallback.js"; import { runDetached } from "../runs/background/runner.js"; import type { JobTracker } from "../runs/background/job-tracker.js"; +import { appendRunEnd } from "../shared/persistence-session-entries.js"; import { classifyWorkflowFailure } from "../shared/workflow-failures.js"; // --------------------------------------------------------------------------- @@ -389,13 +390,39 @@ export function createExtensionRuntime(opts: ExtensionRuntimeOpts = {}): Extensi return { ok: true, stageId: failedStageId }; } + function finalizeResumedActiveBlockedSourceRun(source: RunSnapshot, continuationRunId: string): void { + const errorMessage = source.error ?? source.failureMessage ?? `workflow resumed in new run ${continuationRunId}`; + const metadata = { + ...(source.failureKind !== undefined ? { failureKind: source.failureKind } : {}), + ...(source.failureCode !== undefined ? { failureCode: source.failureCode } : {}), + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", + ...(source.failureMessage !== undefined ? { failureMessage: source.failureMessage } : {}), + ...(source.failedStageId !== undefined ? { failedStageId: source.failedStageId } : {}), + resumable: false, + ...(source.retryAfterMs !== undefined ? { retryAfterMs: source.retryAfterMs } : {}), + } as const; + const recorded = activeStore.recordRunEnd(source.id, "killed", undefined, errorMessage, metadata); + if (recorded && persistence !== undefined) { + appendRunEnd(persistence, { + runId: source.id, + status: "killed", + error: errorMessage, + ...metadata, + ts: Date.now(), + }); + } + } + function resumeFailedRun(sourceRunId: string, stageId?: string, options?: RuntimeDispatchOptions): ResumeFailedRunResult { const source = activeStore.runs().find((run) => run.id === sourceRunId); if (source === undefined) { return { ok: false, reason: "run_not_found", message: `run not found: ${sourceRunId}` }; } - if (source.status !== "failed" || source.endedAt === undefined || source.resumable === false) { - return { ok: false, reason: "not_resumable", message: `run ${sourceRunId} is not a failed resumable workflow run` }; + const isTerminalFailedResumable = source.status === "failed" && source.endedAt !== undefined && source.resumable !== false; + const isActiveBlockedResumable = source.endedAt === undefined && source.resumable === true && source.failureRecoverability === "recoverable"; + if (!isTerminalFailedResumable && !isActiveBlockedResumable) { + return { ok: false, reason: "not_resumable", message: `run ${sourceRunId} is not a resumable workflow run` }; } const def = registry.get(source.name); if (def === undefined) { @@ -415,12 +442,17 @@ export function createExtensionRuntime(opts: ExtensionRuntimeOpts = {}): Extensi ...runOptions({ workflow: def.name, inputs: sourceInputs }, options?.policy), continuation: { source, resumeFromStageId: resolvedStage.stageId }, }); + if (isActiveBlockedResumable) { + finalizeResumedActiveBlockedSourceRun(source, accepted.runId); + } return { ok: true, runId: accepted.runId, sourceRunId: source.id, resumeFromStageId: resolvedStage.stageId, - message: `Resuming failed workflow "${def.name}" from run ${source.id.slice(0, 8)} at stage ${resolvedStage.stageId.slice(0, 8)} (new run ${accepted.runId}).`, + message: isActiveBlockedResumable + ? `Resuming blocked workflow "${def.name}" from run ${source.id.slice(0, 8)} at stage ${resolvedStage.stageId.slice(0, 8)} (new run ${accepted.runId}).` + : `Resuming failed workflow "${def.name}" from run ${source.id.slice(0, 8)} at stage ${resolvedStage.stageId.slice(0, 8)} (new run ${accepted.runId}).`, }; } diff --git a/packages/workflows/src/runs/background/status.ts b/packages/workflows/src/runs/background/status.ts index c19193244..2f0ff9521 100644 --- a/packages/workflows/src/runs/background/status.ts +++ b/packages/workflows/src/runs/background/status.ts @@ -82,6 +82,14 @@ export interface RunDetail { readonly stages: readonly RunSnapshot["stages"][number][]; readonly result?: WorkflowOutputValues; readonly error?: string; + readonly failureKind?: RunSnapshot["failureKind"]; + readonly failureCode?: RunSnapshot["failureCode"]; + readonly failureRecoverability?: RunSnapshot["failureRecoverability"]; + readonly failureDisposition?: RunSnapshot["failureDisposition"]; + readonly failedStageId?: string; + readonly resumable?: boolean; + readonly retryAfterMs?: number; + readonly blockedAt?: number; } export type InspectRunResult = @@ -146,11 +154,26 @@ export function killRun( const previousStatus = run.status; // Abort active executor (no-op if not registered) - opts?.cancellation?.abort(runId, "workflow killed"); - - const recorded = activeStore.recordRunEnd(runId, "killed", undefined, "workflow killed"); + const errorMessage = "workflow killed"; + opts?.cancellation?.abort(runId, errorMessage); + + const metadata = { + failureKind: "cancelled", + failureCode: "cancelled", + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", + failureMessage: errorMessage, + resumable: false, + } as const; + const recorded = activeStore.recordRunEnd(runId, "killed", undefined, errorMessage, metadata); if (recorded && opts?.persistence) { - appendRunEnd(opts.persistence, { runId, status: "killed", ts: Date.now() }); + appendRunEnd(opts.persistence, { + runId, + status: "killed", + error: errorMessage, + ...metadata, + ts: Date.now(), + }); } return { ok: true, runId, previousStatus }; @@ -247,14 +270,29 @@ export function resumeRun( // Return a deep copy of the snapshot for safe consumption const snapshot = structuredClone(run); const resumedCopy = structuredClone(resumed); - if (run.status === "failed" && run.endedAt !== undefined && run.resumable === false) { + if (run.status === "killed" || run.resumable === false) { return { ok: true, runId, snapshot, resumed: resumedCopy, mode: "not_resumable", - message: "This failed workflow is not resumable; inspect the snapshot and rerun the workflow when ready.", + message: "This workflow is not resumable; inspect the snapshot and start a new workflow run when ready.", + }; + } + if ( + run.endedAt === undefined && + run.resumable === true && + run.failureRecoverability === "recoverable" && + run.failedStageId !== undefined + ) { + return { + ok: true, + runId, + snapshot, + resumed: resumedCopy, + mode: resumedCopy.length > 0 ? "paused" : "snapshot", + message: `Workflow is blocked on a recoverable ${run.failureCode ?? run.failureKind ?? "workflow"} failure at stage ${run.failedStageId}; retry/resume after the issue clears.`, }; } return { @@ -441,6 +479,14 @@ export function inspectRun( stages: expandedStages.map((stage) => structuredClone(stage)), result: copy.result, error: copy.error, + failureKind: copy.failureKind, + failureCode: copy.failureCode, + failureRecoverability: copy.failureRecoverability, + failureDisposition: copy.failureDisposition, + failedStageId: copy.failedStageId, + resumable: copy.resumable, + retryAfterMs: copy.retryAfterMs, + blockedAt: copy.blockedAt, }; return { ok: true, runId: copy.id, detail }; diff --git a/packages/workflows/src/runs/foreground/executor.ts b/packages/workflows/src/runs/foreground/executor.ts index 63360ac94..f59d826bf 100644 --- a/packages/workflows/src/runs/foreground/executor.ts +++ b/packages/workflows/src/runs/foreground/executor.ts @@ -52,6 +52,9 @@ import type { RunSnapshot, WorkflowOverlayAdapter, WorkflowFailureKind, + WorkflowFailureCode, + WorkflowFailureRecoverability, + WorkflowFailureDisposition, PendingPrompt, PromptKind, WorkflowChildReplaySnapshot, @@ -82,6 +85,7 @@ import { appendStageStart, appendStageEnd, appendRunEnd, + appendRunBlocked, } from "../../shared/persistence-session-entries.js"; import { buildModelCandidatesFromCatalog, validateWorkflowModels, workflowModelId } from "../shared/model-fallback.js"; import { validateInputs, type ValidationError } from "../shared/validate-inputs.js"; @@ -1134,7 +1138,13 @@ function workflowDetailsFromRun( mode, action: "run", runId: runResult.runId, - status: runResult.status === "killed" ? "killed" : runResult.status === "failed" ? "failed" : "completed", + status: runResult.status === "completed" + ? "completed" + : runResult.status === "failed" + ? "failed" + : runResult.status === "killed" + ? "killed" + : "running", ...(options.context !== undefined ? { context: options.context } : {}), results: [...results], output: runResult.result, @@ -1415,9 +1425,13 @@ function appendRunEndWhenRecorded( readonly result?: WorkflowOutputValues; readonly error?: string; readonly failureKind?: WorkflowFailureKind; + readonly failureCode?: WorkflowFailureCode; + readonly failureRecoverability?: WorkflowFailureRecoverability; + readonly failureDisposition?: WorkflowFailureDisposition; readonly failureMessage?: string; readonly failedStageId?: string; readonly resumable?: boolean; + readonly retryAfterMs?: number; readonly ts: number; }, ): void { @@ -1428,32 +1442,272 @@ function appendRunEndWhenRecorded( interface RunFailureMetadata { readonly errorMessage: string; readonly failureKind: WorkflowFailureKind; + readonly failureCode?: WorkflowFailureCode; + readonly failureRecoverability?: WorkflowFailureRecoverability; + readonly failureDisposition?: WorkflowFailureDisposition; readonly failureMessage: string; readonly failedStageId?: string; readonly resumable: boolean; + readonly retryAfterMs?: number; } function applyFailureToStage(stage: StageSnapshot, failure: WorkflowFailure): void { stage.status = "failed"; stage.error = failure.userMessage; stage.failureKind = failure.kind; + stage.failureCode = failure.code; + stage.failureRecoverability = failure.recoverability; + stage.failureDisposition = failure.disposition; + stage.retryAfterMs = failure.retryAfterMs; stage.failureMessage = failure.message; } -function runFailureMetadata(err: unknown, stages: readonly StageSnapshot[]): RunFailureMetadata { - const classified = classifyWorkflowFailure(err); +function runFailureMetadata( + failure: WorkflowFailure, + stages: readonly StageSnapshot[], +): RunFailureMetadata { const failedStage = stages.find((stage) => stage.status === "failed"); - const failureKind = failedStage?.failureKind ?? classified.kind; + const failureKind = failedStage?.failureKind ?? failure.kind; + const failureCode = failedStage?.failureCode ?? failure.code; + const failureRecoverability = failedStage?.failureRecoverability ?? failure.recoverability; + const failureDisposition = failedStage?.failureDisposition ?? failure.disposition; + const retryAfterMs = failedStage?.retryAfterMs ?? failure.retryAfterMs; return { - errorMessage: classified.userMessage, + errorMessage: failedStage?.error ?? failure.userMessage, failureKind, - failureMessage: failedStage?.failureMessage ?? classified.message, + ...(failureCode !== undefined ? { failureCode } : {}), + failureRecoverability, + failureDisposition, + failureMessage: failedStage?.failureMessage ?? failure.message, ...(failedStage !== undefined ? { failedStageId: failedStage.id } : {}), - resumable: classified.resumable, + resumable: failure.resumable, + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), }; } +interface SelectedRunFailureMetadata extends RunFailureMetadata { + readonly failedStageIds: readonly string[]; +} + +function stageDispositionResumable( + disposition: WorkflowFailureDisposition | undefined, + fallback: boolean, +): boolean { + if (disposition === "terminal_killed") return false; + if (disposition === "active_blocked") return true; + return fallback; +} + +function runFailureMetadataFromStage( + fallbackFailure: WorkflowFailure, + stage: StageSnapshot, +): RunFailureMetadata { + const failureKind = stage.failureKind ?? fallbackFailure.kind; + const failureCode = stage.failureCode ?? fallbackFailure.code; + const failureRecoverability = stage.failureRecoverability ?? fallbackFailure.recoverability; + const failureDisposition = stage.failureDisposition ?? fallbackFailure.disposition; + const retryAfterMs = stage.retryAfterMs ?? fallbackFailure.retryAfterMs; + + return { + errorMessage: stage.error ?? fallbackFailure.userMessage, + failureKind, + ...(failureCode !== undefined ? { failureCode } : {}), + failureRecoverability, + failureDisposition, + failureMessage: stage.failureMessage ?? fallbackFailure.message, + failedStageId: stage.id, + resumable: stageDispositionResumable(failureDisposition, fallbackFailure.resumable), + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }; +} + +function runFailureMetadataFromFailure( + failure: WorkflowFailure, + failedStage: StageSnapshot | undefined, +): RunFailureMetadata { + return { + errorMessage: failedStage?.error ?? failure.userMessage, + failureKind: failure.kind, + ...(failure.code !== undefined ? { failureCode: failure.code } : {}), + failureRecoverability: failure.recoverability, + failureDisposition: failure.disposition, + failureMessage: failedStage?.failureMessage ?? failure.message, + ...(failedStage !== undefined ? { failedStageId: failedStage.id } : {}), + resumable: failure.resumable, + ...(failure.retryAfterMs !== undefined ? { retryAfterMs: failure.retryAfterMs } : {}), + }; +} + +function executorAggregateErrorItems(error: unknown): readonly unknown[] { + const nativeErrors = error instanceof AggregateError ? error.errors as unknown : undefined; + const errors = nativeErrors ?? (error !== null && typeof error === "object" + ? (error as Record)["errors"] + : undefined); + return Array.isArray(errors) ? errors : []; +} + +function isAggregateWrapper(error: unknown): boolean { + return executorAggregateErrorItems(error).length > 0; +} + +function aggregateInnerFailures( + error: unknown, + classifyFailure: (error: unknown) => WorkflowFailure, +): readonly WorkflowFailure[] { + return executorAggregateErrorItems(error).map((innerError) => classifyFailure(innerError)); +} + +type StageFailureCandidate = { + readonly source: "stage"; + readonly stage: StageSnapshot; + readonly disposition: WorkflowFailureDisposition; + readonly recoverability: WorkflowFailureRecoverability; +}; + +type AggregateFailureCandidate = { + readonly source: "aggregate"; + readonly failure: WorkflowFailure; + readonly disposition: WorkflowFailureDisposition; + readonly recoverability: WorkflowFailureRecoverability; +}; + +type OuterFailureCandidate = { + readonly source: "outer"; + readonly failure: WorkflowFailure; + readonly disposition: WorkflowFailureDisposition; + readonly recoverability: WorkflowFailureRecoverability; +}; + +type FailureCandidate = StageFailureCandidate | AggregateFailureCandidate | OuterFailureCandidate; + +function stageFailureCandidate(stage: StageSnapshot): StageFailureCandidate { + return { + source: "stage", + stage, + disposition: stage.failureDisposition ?? "terminal_failed", + recoverability: stage.failureRecoverability ?? "unknown", + }; +} + +function aggregateFailureCandidate(failure: WorkflowFailure): AggregateFailureCandidate { + return { + source: "aggregate", + failure, + disposition: failure.disposition, + recoverability: failure.recoverability, + }; +} + +function outerFailureCandidate(failure: WorkflowFailure): OuterFailureCandidate { + return { + source: "outer", + failure, + disposition: failure.disposition, + recoverability: failure.recoverability, + }; +} + +function isRecoverableActiveBlockedCandidate(candidate: FailureCandidate): boolean { + return candidate.disposition === "active_blocked" && candidate.recoverability === "recoverable"; +} + +function runFailureMetadataFromCandidate( + fallbackFailure: WorkflowFailure, + candidate: FailureCandidate, + thrownError: unknown, +): RunFailureMetadata { + let metadata: RunFailureMetadata; + switch (candidate.source) { + case "stage": + metadata = runFailureMetadataFromStage(fallbackFailure, candidate.stage); + break; + case "aggregate": + case "outer": + metadata = runFailureMetadataFromFailure(candidate.failure, undefined); + break; + } + + if (candidate.disposition === "terminal_failed" && isAggregateWrapper(thrownError)) { + return { ...metadata, errorMessage: fallbackFailure.userMessage }; + } + + return metadata; +} + +function failedStageIdsForCandidate( + candidate: FailureCandidate, + failedStages: readonly StageSnapshot[], +): readonly string[] { + switch (candidate.source) { + case "aggregate": + return failedStages.map((stage) => stage.id); + case "outer": + return []; + case "stage": + return failedStages + .filter((stage) => (stage.failureDisposition ?? "terminal_failed") === candidate.disposition) + .map((stage) => stage.id); + } +} + +function selectedMetadata( + metadata: RunFailureMetadata, + failedStageIds: readonly string[], +): SelectedRunFailureMetadata { + return { + ...metadata, + failedStageIds, + }; +} + +function selectRunFailureDisposition(input: { + readonly outerFailure: WorkflowFailure; + readonly thrownError: unknown; + readonly stages: readonly StageSnapshot[]; + readonly classifyFailure: (error: unknown) => WorkflowFailure; +}): SelectedRunFailureMetadata { + const failedStages = input.stages.filter((stage) => stage.status === "failed"); + const failedStageIds = failedStages.map((stage) => stage.id); + const aggregateFailures = aggregateInnerFailures(input.thrownError, input.classifyFailure); + const candidates: readonly FailureCandidate[] = [ + ...failedStages.map(stageFailureCandidate), + ...aggregateFailures.map(aggregateFailureCandidate), + outerFailureCandidate(input.outerFailure), + ]; + // Candidate precedence mirrors lifecycle severity: terminal killed is non-resumable + // and wins first, terminal failed wins over recoverable blocks, and active-blocked + // is only preserved when every observed failure is recoverable active-blocked. + const terminalKilledCandidate = candidates.find((candidate) => candidate.disposition === "terminal_killed"); + if (terminalKilledCandidate !== undefined) { + return selectedMetadata( + runFailureMetadataFromCandidate(input.outerFailure, terminalKilledCandidate, input.thrownError), + failedStageIdsForCandidate(terminalKilledCandidate, failedStages), + ); + } + + const terminalFailedCandidate = candidates.find((candidate) => candidate.disposition === "terminal_failed"); + if (terminalFailedCandidate !== undefined) { + return selectedMetadata( + runFailureMetadataFromCandidate(input.outerFailure, terminalFailedCandidate, input.thrownError), + failedStageIdsForCandidate(terminalFailedCandidate, failedStages), + ); + } + + const recoverableBlockedCandidate = candidates.find(isRecoverableActiveBlockedCandidate); + if ( + recoverableBlockedCandidate !== undefined && + candidates.every(isRecoverableActiveBlockedCandidate) + ) { + return selectedMetadata( + runFailureMetadataFromCandidate(input.outerFailure, recoverableBlockedCandidate, input.thrownError), + failedStageIds, + ); + } + + return selectedMetadata(runFailureMetadata(input.outerFailure, input.stages), failedStageIds); +} + function stageReplayFields(stage: StageSnapshot): Partial> { return { ...(stage.replayKey !== undefined ? { replayKey: stage.replayKey } : {}), @@ -1665,6 +1919,9 @@ function finalizeKilled( const errorMessage = "workflow killed"; const metadata = { failureKind: "cancelled" as const, + failureCode: "cancelled" as const, + failureRecoverability: "non_recoverable" as const, + failureDisposition: "terminal_killed" as const, failureMessage: errorMessage, resumable: false, }; @@ -1685,6 +1942,80 @@ function finalizeKilled( }; } +function finalizeKilledByFailure( + runId: string, + runSnapshot: RunSnapshot, + activeStore: Store, + persistence: WorkflowPersistencePort | undefined, + onRunEnd: RunOpts["onRunEnd"], + metadata: RunFailureMetadata, +): RunResult { + const recorded = activeStore.recordRunEnd(runId, "killed", undefined, metadata.errorMessage, metadata); + onRunEnd?.(runId, "killed", undefined, metadata.errorMessage); + appendRunEndWhenRecorded(persistence, recorded, { + runId, + status: "killed", + error: metadata.errorMessage, + failureKind: metadata.failureKind, + ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}), + ...(metadata.failureRecoverability !== undefined ? { failureRecoverability: metadata.failureRecoverability } : {}), + ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}), + failureMessage: metadata.failureMessage, + ...(metadata.failedStageId !== undefined ? { failedStageId: metadata.failedStageId } : {}), + resumable: false, + ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}), + ts: Date.now(), + }); + return { + runId, + status: "killed", + error: metadata.errorMessage, + stages: [...runSnapshot.stages], + }; +} + +function recordActiveBlockedFailure( + runId: string, + runSnapshot: RunSnapshot, + activeStore: Store, + persistence: WorkflowPersistencePort | undefined, + metadata: RunFailureMetadata & { readonly failureRecoverability: "recoverable"; readonly failedStageId: string }, +): RunResult { + const blockedAt = Date.now(); + const recorded = activeStore.recordRunBlocked(runId, metadata.errorMessage, { + failureKind: metadata.failureKind, + ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}), + failureRecoverability: "recoverable", + ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}), + failureMessage: metadata.failureMessage, + failedStageId: metadata.failedStageId, + resumable: true, + ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}), + blockedAt, + }); + if (recorded && persistence !== undefined) { + appendRunBlocked(persistence, { + runId, + failedStageId: metadata.failedStageId, + error: metadata.errorMessage, + failureKind: metadata.failureKind, + ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}), + failureMessage: metadata.failureMessage, + failureRecoverability: "recoverable", + ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}), + ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}), + resumable: true, + ts: blockedAt, + }); + } + return { + runId, + status: "running", + error: metadata.errorMessage, + stages: [...runSnapshot.stages], + }; +} + // --------------------------------------------------------------------------- // Main executor // --------------------------------------------------------------------------- @@ -1949,6 +2280,15 @@ export async function run( } : {}), }; + const classifiedFailures = new Map(); + const classifyExecutorFailure = (error: unknown): WorkflowFailure => { + const cached = classifiedFailures.get(error); + if (cached !== undefined) return cached; + const classified = classifyWorkflowFailure(error); + classifiedFailures.set(error, classified); + return classified; + }; + activeStore.recordRunStart(runSnapshot); // When the caller already has a controller registered (the detached runner // pre-registers before calling run() so abort() can hit the run during @@ -2061,6 +2401,13 @@ export async function run( activeStore.recordStageBlocked(runId, stage.id, blockedBy); }; + const blockKnownNonTerminalDescendants = (failedStageId: string): void => { + for (const descendant of descendantsOf(failedStageId)) { + if (isTerminalStage(descendant) || descendant.status === "paused" || descendant.status === "blocked") continue; + blockStageUntilCascadeRelease(descendant, failedStageId); + } + }; + const markCascadePaused = (stageId: string, ownerStageId: string): void => { let owners = cascadePauseOwners.get(stageId); if (!owners) { @@ -2226,7 +2573,11 @@ export async function run( durationMs: stageSnapshot.durationMs, ...(stageSnapshot.error !== undefined ? { error: stageSnapshot.error } : {}), ...(stageSnapshot.failureKind !== undefined ? { failureKind: stageSnapshot.failureKind } : {}), + ...(stageSnapshot.failureCode !== undefined ? { failureCode: stageSnapshot.failureCode } : {}), + ...(stageSnapshot.failureRecoverability !== undefined ? { failureRecoverability: stageSnapshot.failureRecoverability } : {}), + ...(stageSnapshot.failureDisposition !== undefined ? { failureDisposition: stageSnapshot.failureDisposition } : {}), ...(stageSnapshot.failureMessage !== undefined ? { failureMessage: stageSnapshot.failureMessage } : {}), + ...(stageSnapshot.retryAfterMs !== undefined ? { retryAfterMs: stageSnapshot.retryAfterMs } : {}), ...(stageSnapshot.result !== undefined && stageSnapshot.status === "completed" ? { summary: stageSnapshot.result } : {}), ...stageReplayFields(stageSnapshot), ...(stageSnapshot.workflowChild !== undefined ? { workflowChild: stageSnapshot.workflowChild } : {}), @@ -2246,10 +2597,7 @@ export async function run( stageSnapshot.result = summaryOrError; if (workflowChild !== undefined) stageSnapshot.workflowChild = workflowChild; } else { - const failure = classifyWorkflowFailure(failureError); - stageSnapshot.error = failure.userMessage; - stageSnapshot.failureKind = failure.kind; - stageSnapshot.failureMessage = failure.message; + applyFailureToStage(stageSnapshot, classifyExecutorFailure(failureError)); } stageSnapshot.endedAt = Date.now(); stageSnapshot.durationMs = elapsedStageMs(stageSnapshot, stageSnapshot.endedAt); @@ -2364,7 +2712,11 @@ export async function run( durationMs: stageSnapshot.durationMs, ...(stageSnapshot.error !== undefined ? { error: stageSnapshot.error } : {}), ...(stageSnapshot.failureKind !== undefined ? { failureKind: stageSnapshot.failureKind } : {}), + ...(stageSnapshot.failureCode !== undefined ? { failureCode: stageSnapshot.failureCode } : {}), + ...(stageSnapshot.failureRecoverability !== undefined ? { failureRecoverability: stageSnapshot.failureRecoverability } : {}), + ...(stageSnapshot.failureDisposition !== undefined ? { failureDisposition: stageSnapshot.failureDisposition } : {}), ...(stageSnapshot.failureMessage !== undefined ? { failureMessage: stageSnapshot.failureMessage } : {}), + ...(stageSnapshot.retryAfterMs !== undefined ? { retryAfterMs: stageSnapshot.retryAfterMs } : {}), ...(stageSnapshot.skippedReason !== undefined ? { skippedReason: stageSnapshot.skippedReason } : {}), ...stageReplayFields(stageSnapshot), }); @@ -2432,7 +2784,7 @@ export async function run( stageSnapshot.skippedReason = "run-aborted"; finalizePromptStage("skipped"); } else { - applyFailureToStage(stageSnapshot, classifyWorkflowFailure(err)); + applyFailureToStage(stageSnapshot, classifyExecutorFailure(err)); finalizePromptStage("failed"); } throw err; @@ -2828,7 +3180,11 @@ export async function run( durationMs: stageSnapshot.durationMs, ...(stageSnapshot.error !== undefined ? { error: stageSnapshot.error } : {}), ...(stageSnapshot.failureKind !== undefined ? { failureKind: stageSnapshot.failureKind } : {}), + ...(stageSnapshot.failureCode !== undefined ? { failureCode: stageSnapshot.failureCode } : {}), + ...(stageSnapshot.failureRecoverability !== undefined ? { failureRecoverability: stageSnapshot.failureRecoverability } : {}), + ...(stageSnapshot.failureDisposition !== undefined ? { failureDisposition: stageSnapshot.failureDisposition } : {}), ...(stageSnapshot.failureMessage !== undefined ? { failureMessage: stageSnapshot.failureMessage } : {}), + ...(stageSnapshot.retryAfterMs !== undefined ? { retryAfterMs: stageSnapshot.retryAfterMs } : {}), ...(stageSnapshot.skippedReason !== undefined ? { skippedReason: stageSnapshot.skippedReason } : {}), ...(stageSnapshot.result !== undefined && stageSnapshot.status === "completed" ? { summary: stageSnapshot.result } : {}), ...stageReplayFields(stageSnapshot), @@ -3066,7 +3422,7 @@ export async function run( return result; } catch (err) { if (!ownController.signal.aborted && !skippedForParallelFailFast) { - applyFailureToStage(stageSnapshot, classifyWorkflowFailure(err)); + applyFailureToStage(stageSnapshot, classifyExecutorFailure(err)); } throw err; } finally { @@ -3429,7 +3785,40 @@ export async function run( return finalizeKilled(runId, runSnapshot, activeStore, opts.persistence, opts.onRunEnd); } - const metadata = runFailureMetadata(err, runSnapshot.stages); + const failure = classifyExecutorFailure(err); + const metadata = selectRunFailureDisposition({ + outerFailure: failure, + thrownError: err, + stages: runSnapshot.stages, + classifyFailure: classifyExecutorFailure, + }); + + if (metadata.failureDisposition === "terminal_killed") { + for (const failedStageId of metadata.failedStageIds) { + blockKnownNonTerminalDescendants(failedStageId); + } + return finalizeKilledByFailure(runId, runSnapshot, activeStore, opts.persistence, opts.onRunEnd, { + ...metadata, + resumable: false, + }); + } + + if ( + metadata.failureDisposition === "active_blocked" && + metadata.failedStageId !== undefined && + metadata.failureRecoverability === "recoverable" + ) { + for (const failedStageId of metadata.failedStageIds) { + blockKnownNonTerminalDescendants(failedStageId); + } + return recordActiveBlockedFailure(runId, runSnapshot, activeStore, opts.persistence, { + ...metadata, + failureRecoverability: "recoverable", + failedStageId: metadata.failedStageId, + resumable: true, + }); + } + const recorded = activeStore.recordRunEnd(runId, "failed", undefined, metadata.errorMessage, metadata); opts.onRunEnd?.(runId, "failed", undefined, metadata.errorMessage); @@ -3438,9 +3827,13 @@ export async function run( status: "failed", error: metadata.errorMessage, failureKind: metadata.failureKind, + ...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}), + ...(metadata.failureRecoverability !== undefined ? { failureRecoverability: metadata.failureRecoverability } : {}), + ...(metadata.failureDisposition !== undefined ? { failureDisposition: metadata.failureDisposition } : {}), failureMessage: metadata.failureMessage, ...(metadata.failedStageId !== undefined ? { failedStageId: metadata.failedStageId } : {}), resumable: metadata.resumable, + ...(metadata.retryAfterMs !== undefined ? { retryAfterMs: metadata.retryAfterMs } : {}), ts: Date.now(), }); diff --git a/packages/workflows/src/shared/persistence-restore.ts b/packages/workflows/src/shared/persistence-restore.ts index 558a272c0..9bc05e6a4 100644 --- a/packages/workflows/src/shared/persistence-restore.ts +++ b/packages/workflows/src/shared/persistence-restore.ts @@ -7,11 +7,24 @@ */ import type { Store } from "./store.js"; -import type { RunSnapshot, StageSnapshot, StageStatus, WorkflowChildReplaySnapshot } from "./store-types.js"; +import type { + RunSnapshot, + StageSnapshot, + StageStatus, + WorkflowChildReplaySnapshot, + WorkflowFailureCode, + WorkflowFailureDisposition, + WorkflowFailureKind, +} from "./store-types.js"; import type { WorkflowInputValues, WorkflowOutputValues } from "./types.js"; import { workflowSerializableObjectSchema } from "./serializable.js"; import { Value } from "typebox/value"; -import { isWorkflowFailureKind } from "./workflow-failures.js"; +import { + isWorkflowFailureCode, + isWorkflowFailureDisposition, + isWorkflowFailureKind, + isWorkflowFailureRecoverability, +} from "./workflow-failures.js"; // --------------------------------------------------------------------------- // Config option @@ -49,6 +62,19 @@ export interface InFlightRun { readonly stageIds: readonly string[]; } +interface RestoredRunBlockedMetadata { + readonly failedStageId: string; + readonly error: string; + readonly failureKind: WorkflowFailureKind; + readonly failureCode?: WorkflowFailureCode; + readonly failureRecoverability: "recoverable"; + readonly failureDisposition?: WorkflowFailureDisposition; + readonly failureMessage?: string; + readonly retryAfterMs?: number; + readonly resumable: true; + readonly ts: number; +} + // --------------------------------------------------------------------------- // Scan logic // --------------------------------------------------------------------------- @@ -170,7 +196,37 @@ export function restoreOnSessionStart( for (const run of inFlight) { const runMeta = findRunStartMetadata(sessionEntries, run.runId); - const stages = _buildStageSnapshots(sessionEntries, run.runId); + const blockedMeta = findRunBlockedMetadata(sessionEntries, run.runId); + const stages = _buildStageSnapshots(sessionEntries, run.runId, blockedMeta); + + if (blockedMeta !== undefined) { + const runSnapshot: RunSnapshot = { + id: run.runId, + name: run.name, + inputs: run.inputs, + status: "running", + stages, + startedAt: run.startTs, + ...(runMeta.parentRunId !== undefined ? { parentRunId: runMeta.parentRunId } : {}), + ...(runMeta.parentStageId !== undefined ? { parentStageId: runMeta.parentStageId } : {}), + ...(runMeta.rootRunId !== undefined ? { rootRunId: runMeta.rootRunId } : {}), + ...(runMeta.resumedFromRunId !== undefined ? { resumedFromRunId: runMeta.resumedFromRunId } : {}), + ...(runMeta.resumeFromStageId !== undefined ? { resumeFromStageId: runMeta.resumeFromStageId } : {}), + }; + store.recordRunStart(runSnapshot); + store.recordRunBlocked(run.runId, blockedMeta.error, { + failureKind: blockedMeta.failureKind, + ...(blockedMeta.failureCode !== undefined ? { failureCode: blockedMeta.failureCode } : {}), + failureRecoverability: "recoverable", + ...(blockedMeta.failureDisposition !== undefined ? { failureDisposition: blockedMeta.failureDisposition } : {}), + ...(blockedMeta.failureMessage !== undefined ? { failureMessage: blockedMeta.failureMessage } : {}), + failedStageId: blockedMeta.failedStageId, + resumable: true, + ...(blockedMeta.retryAfterMs !== undefined ? { retryAfterMs: blockedMeta.retryAfterMs } : {}), + blockedAt: blockedMeta.ts, + }); + continue; + } if (config.resumeInFlight === "auto") { // Re-hydrate the run into the store as "running" @@ -223,6 +279,7 @@ export function restoreOnSessionStart( function _buildStageSnapshots( entries: readonly SessionEntry[], runId: string, + blockedMeta?: RestoredRunBlockedMetadata, ): StageSnapshot[] { const stageMap = new Map(); const endedStages = new Set(); @@ -256,6 +313,10 @@ function _buildStageSnapshots( const summary = entry.payload["summary"]; const error = entry.payload["error"]; const failureKind = entry.payload["failureKind"]; + const failureCode = entry.payload["failureCode"]; + const failureRecoverability = entry.payload["failureRecoverability"]; + const failureDisposition = entry.payload["failureDisposition"]; + const retryAfterMs = entry.payload["retryAfterMs"]; const failureMessage = entry.payload["failureMessage"]; const skippedReason = entry.payload["skippedReason"]; if (typeof stageId !== "string") continue; @@ -267,6 +328,10 @@ function _buildStageSnapshots( if (typeof summary === "string") snap.result = summary; if (typeof error === "string") snap.error = error; if (typeof failureKind === "string" && isWorkflowFailureKind(failureKind)) snap.failureKind = failureKind; + if (typeof failureCode === "string" && isWorkflowFailureCode(failureCode)) snap.failureCode = failureCode; + if (typeof failureRecoverability === "string" && isWorkflowFailureRecoverability(failureRecoverability)) snap.failureRecoverability = failureRecoverability; + if (typeof failureDisposition === "string" && isWorkflowFailureDisposition(failureDisposition)) snap.failureDisposition = failureDisposition; + if (typeof retryAfterMs === "number") snap.retryAfterMs = retryAfterMs; if (typeof failureMessage === "string") snap.failureMessage = failureMessage; if (typeof skippedReason === "string") snap.skippedReason = skippedReason; Object.assign(snap, replayMetadata(entry.payload), workflowChildMetadata(entry.payload)); @@ -274,9 +339,12 @@ function _buildStageSnapshots( } } - // Mark any stage that didn't get an end entry as "failed" (crashed) - for (const [stageId, snap] of stageMap) { - if (!endedStages.has(stageId)) { + if (blockedMeta !== undefined) { + restoreBlockedStageState(stageMap, endedStages, blockedMeta); + } else { + // Mark any stage that didn't get an end entry as crashed. + for (const [stageId, snap] of stageMap) { + if (endedStages.has(stageId)) continue; snap.status = "failed"; snap.error = "Stage did not complete — process was interrupted."; } @@ -285,6 +353,57 @@ function _buildStageSnapshots( return [...stageMap.values()]; } +function hasRestoredAncestor( + stageMap: ReadonlyMap, + stage: StageSnapshot, + ancestorId: string, +): boolean { + const queue = [...stage.parentIds]; + const seen = new Set(); + + while (queue.length > 0) { + const next = queue.shift(); + if (next === undefined || seen.has(next)) continue; + if (next === ancestorId) return true; + seen.add(next); + queue.push(...(stageMap.get(next)?.parentIds ?? [])); + } + + return false; +} + +function markRestoredBlockedFailureStage( + snap: StageSnapshot, + blockedMeta: RestoredRunBlockedMetadata, +): void { + snap.status = "failed"; + snap.error = blockedMeta.error; + snap.failureKind = blockedMeta.failureKind; + snap.failureCode = blockedMeta.failureCode; + snap.failureRecoverability = blockedMeta.failureRecoverability; + snap.failureDisposition = blockedMeta.failureDisposition; + snap.failureMessage = blockedMeta.failureMessage; + snap.retryAfterMs = blockedMeta.retryAfterMs; +} + +function restoreBlockedStageState( + stageMap: Map, + endedStages: ReadonlySet, + blockedMeta: RestoredRunBlockedMetadata, +): void { + for (const [stageId, snap] of stageMap) { + if (endedStages.has(stageId)) continue; + if (stageId === blockedMeta.failedStageId) { + markRestoredBlockedFailureStage(snap, blockedMeta); + continue; + } + if (hasRestoredAncestor(stageMap, snap, blockedMeta.failedStageId)) { + snap.status = "blocked"; + snap.blockedByStageId = blockedMeta.failedStageId; + } + } +} + function replayMetadata(payload: Record): Pick { const replayKey = payload["replayKey"]; const replayedFromStageId = payload["replayedFromStageId"]; @@ -361,12 +480,61 @@ function restoreStageStatus(status: unknown): StageStatus { case "completed": case "failed": case "skipped": + case "blocked": return status; default: return "failed"; } } +function numericRetryAfterMs(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function findRunBlockedMetadata( + entries: readonly SessionEntry[], + runId: string, +): RestoredRunBlockedMetadata | undefined { + let latest: RestoredRunBlockedMetadata | undefined; + for (const entry of entries) { + if (entry.type !== "workflow.run.blocked" || entry.payload["runId"] !== runId) continue; + const failedStageId = entry.payload["failedStageId"]; + const error = entry.payload["error"]; + const failureKind = entry.payload["failureKind"]; + const failureCode = entry.payload["failureCode"]; + const failureRecoverability = entry.payload["failureRecoverability"]; + const failureDisposition = entry.payload["failureDisposition"]; + const failureMessage = entry.payload["failureMessage"]; + const retryAfterMs = numericRetryAfterMs(entry.payload["retryAfterMs"]); + const resumable = entry.payload["resumable"]; + const ts = entry.payload["ts"]; + if ( + typeof failedStageId !== "string" || + typeof error !== "string" || + typeof failureKind !== "string" || + !isWorkflowFailureKind(failureKind) || + failureRecoverability !== "recoverable" || + resumable !== true || + typeof ts !== "number" + ) { + continue; + } + latest = { + failedStageId, + error, + failureKind, + ...(typeof failureCode === "string" && isWorkflowFailureCode(failureCode) ? { failureCode } : {}), + failureRecoverability: "recoverable", + ...(typeof failureDisposition === "string" && isWorkflowFailureDisposition(failureDisposition) ? { failureDisposition } : {}), + ...(typeof failureMessage === "string" ? { failureMessage } : {}), + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + resumable: true, + ts, + }; + } + return latest; +} + function restoreTerminalRuns(entries: readonly SessionEntry[], store: Store): void { const started = new Map; readonly startTs: number }>(); const ended = new Map>(); @@ -416,6 +584,10 @@ function restoreTerminalRuns(entries: readonly SessionEntry[], store: Store): vo const error = end["error"]; const failureKind = end["failureKind"]; + const failureCode = end["failureCode"]; + const failureRecoverability = end["failureRecoverability"]; + const failureDisposition = end["failureDisposition"]; + const retryAfterMs = numericRetryAfterMs(end["retryAfterMs"]); const failureMessage = end["failureMessage"]; const failedStageId = end["failedStageId"]; const resumable = end["resumable"]; @@ -426,6 +598,10 @@ function restoreTerminalRuns(entries: readonly SessionEntry[], store: Store): vo typeof error === "string" ? error : undefined, { ...(typeof failureKind === "string" && isWorkflowFailureKind(failureKind) ? { failureKind } : {}), + ...(typeof failureCode === "string" && isWorkflowFailureCode(failureCode) ? { failureCode } : {}), + ...(typeof failureRecoverability === "string" && isWorkflowFailureRecoverability(failureRecoverability) ? { failureRecoverability } : {}), + ...(typeof failureDisposition === "string" && isWorkflowFailureDisposition(failureDisposition) ? { failureDisposition } : {}), + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), ...(typeof failureMessage === "string" ? { failureMessage } : {}), ...(typeof failedStageId === "string" ? { failedStageId } : {}), ...(typeof resumable === "boolean" ? { resumable } : {}), diff --git a/packages/workflows/src/shared/persistence-session-entries.ts b/packages/workflows/src/shared/persistence-session-entries.ts index 0d3e15b89..f6b08836d 100644 --- a/packages/workflows/src/shared/persistence-session-entries.ts +++ b/packages/workflows/src/shared/persistence-session-entries.ts @@ -7,6 +7,11 @@ */ import type { WorkflowInputValues, WorkflowOutputValues } from "./types.js"; +import type { + WorkflowFailureCode, + WorkflowFailureDisposition, + WorkflowFailureKind, +} from "./store-types.js"; // --------------------------------------------------------------------------- // Structural API type (subset of ExtensionAPI needed here) @@ -76,7 +81,11 @@ export interface StageEndPayload { readonly summary?: string; readonly error?: string; readonly failureKind?: string; + readonly failureCode?: string; + readonly failureRecoverability?: string; + readonly failureDisposition?: string; readonly failureMessage?: string; + readonly retryAfterMs?: number; readonly skippedReason?: string; readonly replayKey?: string; readonly replayedFromStageId?: string; @@ -90,9 +99,27 @@ export interface RunEndPayload { readonly result?: WorkflowOutputValues; readonly error?: string; readonly failureKind?: string; + readonly failureCode?: string; + readonly failureRecoverability?: string; + readonly failureDisposition?: string; readonly failureMessage?: string; readonly failedStageId?: string; readonly resumable?: boolean; + readonly retryAfterMs?: number; + readonly ts: number; +} + +export interface RunBlockedPayload { + readonly runId: string; + readonly failedStageId: string; + readonly error: string; + readonly failureKind: WorkflowFailureKind; + readonly failureCode?: WorkflowFailureCode; + readonly failureMessage?: string; + readonly failureRecoverability: "recoverable"; + readonly failureDisposition?: WorkflowFailureDisposition; + readonly retryAfterMs?: number; + readonly resumable: true; readonly ts: number; } @@ -165,7 +192,11 @@ export function appendStageEnd( ...(payload.summary !== undefined ? { summary: payload.summary } : {}), ...(payload.error !== undefined ? { error: payload.error } : {}), ...(payload.failureKind !== undefined ? { failureKind: payload.failureKind } : {}), + ...(payload.failureCode !== undefined ? { failureCode: payload.failureCode } : {}), + ...(payload.failureRecoverability !== undefined ? { failureRecoverability: payload.failureRecoverability } : {}), + ...(payload.failureDisposition !== undefined ? { failureDisposition: payload.failureDisposition } : {}), ...(payload.failureMessage !== undefined ? { failureMessage: payload.failureMessage } : {}), + ...(payload.retryAfterMs !== undefined ? { retryAfterMs: payload.retryAfterMs } : {}), ...(payload.skippedReason !== undefined ? { skippedReason: payload.skippedReason } : {}), ...(payload.replayKey !== undefined ? { replayKey: payload.replayKey } : {}), ...(payload.replayedFromStageId !== undefined ? { replayedFromStageId: payload.replayedFromStageId } : {}), @@ -180,18 +211,57 @@ export function appendStageEnd( } } +function sanitizeTerminalRunEndPayload(payload: RunEndPayload): RunEndPayload { + if (payload.status === "killed") { + return { + ...payload, + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", + resumable: false, + }; + } + + if (payload.failureDisposition !== "active_blocked") return payload; + const sanitized = { ...payload }; + delete (sanitized as { failureDisposition?: string }).failureDisposition; + return sanitized; +} + /** Appends a `workflow.run.end` entry. */ export function appendRunEnd(api: PersistenceAPI, payload: RunEndPayload): void { if (typeof api.appendEntry !== "function") return; + const terminalPayload = sanitizeTerminalRunEndPayload(payload); api.appendEntry("workflow.run.end", { + runId: terminalPayload.runId, + status: terminalPayload.status, + ...(terminalPayload.result !== undefined ? { result: terminalPayload.result } : {}), + ...(terminalPayload.error !== undefined ? { error: terminalPayload.error } : {}), + ...(terminalPayload.failureKind !== undefined ? { failureKind: terminalPayload.failureKind } : {}), + ...(terminalPayload.failureCode !== undefined ? { failureCode: terminalPayload.failureCode } : {}), + ...(terminalPayload.failureRecoverability !== undefined ? { failureRecoverability: terminalPayload.failureRecoverability } : {}), + ...(terminalPayload.failureDisposition !== undefined ? { failureDisposition: terminalPayload.failureDisposition } : {}), + ...(terminalPayload.failureMessage !== undefined ? { failureMessage: terminalPayload.failureMessage } : {}), + ...(terminalPayload.failedStageId !== undefined ? { failedStageId: terminalPayload.failedStageId } : {}), + ...(terminalPayload.resumable !== undefined ? { resumable: terminalPayload.resumable } : {}), + ...(terminalPayload.retryAfterMs !== undefined ? { retryAfterMs: terminalPayload.retryAfterMs } : {}), + ts: terminalPayload.ts, + }); +} + +/** Appends a `workflow.run.blocked` entry for active recoverable failures. */ +export function appendRunBlocked(api: PersistenceAPI, payload: RunBlockedPayload): void { + if (typeof api.appendEntry !== "function") return; + api.appendEntry("workflow.run.blocked", { runId: payload.runId, - status: payload.status, - ...(payload.result !== undefined ? { result: payload.result } : {}), - ...(payload.error !== undefined ? { error: payload.error } : {}), - ...(payload.failureKind !== undefined ? { failureKind: payload.failureKind } : {}), + failedStageId: payload.failedStageId, + error: payload.error, + failureKind: payload.failureKind, + ...(payload.failureCode !== undefined ? { failureCode: payload.failureCode } : {}), ...(payload.failureMessage !== undefined ? { failureMessage: payload.failureMessage } : {}), - ...(payload.failedStageId !== undefined ? { failedStageId: payload.failedStageId } : {}), - ...(payload.resumable !== undefined ? { resumable: payload.resumable } : {}), + failureRecoverability: payload.failureRecoverability, + ...(payload.failureDisposition !== undefined ? { failureDisposition: payload.failureDisposition } : {}), + ...(payload.retryAfterMs !== undefined ? { retryAfterMs: payload.retryAfterMs } : {}), + resumable: payload.resumable, ts: payload.ts, }); } diff --git a/packages/workflows/src/shared/store-types.ts b/packages/workflows/src/shared/store-types.ts index 386a16350..47f54d2a3 100644 --- a/packages/workflows/src/shared/store-types.ts +++ b/packages/workflows/src/shared/store-types.ts @@ -17,6 +17,19 @@ export type StageStatus = | "skipped"; export type WorkflowFailureKind = "auth" | "rate_limit" | "provider" | "cancelled" | "unknown"; +export type WorkflowFailureRecoverability = "recoverable" | "non_recoverable" | "unknown"; +export type WorkflowFailureDisposition = "active_blocked" | "terminal_killed" | "terminal_failed"; +export type WorkflowFailureCode = + | "login_required" + | "missing_api_key" + | "invalid_api_key" + | "forbidden_config" + | "unknown_model" + | "rate_limited" + | "quota_limited" + | "provider_unavailable" + | "cancelled" + | "unknown"; /** * Human-in-the-loop prompt kind. Mirrors the four `WorkflowUIContext` methods. @@ -127,6 +140,14 @@ export interface StageSnapshot { error?: string; /** Structured workflow failure category for failed stages. */ failureKind?: WorkflowFailureKind; + /** Specific additive workflow failure code within `failureKind`. */ + failureCode?: WorkflowFailureCode; + /** Whether retry/resume can recover this failed stage without a workflow rerun. */ + failureRecoverability?: WorkflowFailureRecoverability; + /** Executor lifecycle disposition chosen for the failed stage. */ + failureDisposition?: WorkflowFailureDisposition; + /** Optional provider retry hint in milliseconds. Informational; blocked stages resume only via explicit user action. */ + retryAfterMs?: number; /** Original unsanitized error text when different from `error`. */ failureMessage?: string; /** Reason for stages skipped by fail-fast/cascade handling. */ @@ -218,6 +239,16 @@ export interface RunSnapshot { error?: string; /** Structured workflow failure category for failed runs. */ failureKind?: WorkflowFailureKind; + /** Specific additive workflow failure code within `failureKind`. */ + failureCode?: WorkflowFailureCode; + /** Whether retry/resume can recover this run without a workflow rerun. */ + failureRecoverability?: WorkflowFailureRecoverability; + /** Executor lifecycle disposition chosen for this failure. */ + failureDisposition?: WorkflowFailureDisposition; + /** Optional provider retry hint in milliseconds. Informational; blocked runs resume only via explicit user action. */ + retryAfterMs?: number; + /** Timestamp when an active run was blocked by a recoverable workflow failure. */ + blockedAt?: number; /** Original unsanitized error text when different from `error`. */ failureMessage?: string; failedStageId?: string; diff --git a/packages/workflows/src/shared/store.ts b/packages/workflows/src/shared/store.ts index e7977e6ef..7099596d3 100644 --- a/packages/workflows/src/shared/store.ts +++ b/packages/workflows/src/shared/store.ts @@ -16,6 +16,9 @@ import type { RunStatus, StageStatus, WorkflowFailureKind, + WorkflowFailureCode, + WorkflowFailureRecoverability, + WorkflowFailureDisposition, WorkflowNotice, WorkflowChildRunRef, } from "./store-types.js"; @@ -43,9 +46,55 @@ function cannotPause(status: StageStatus): boolean { export interface RunEndMetadata { readonly failureKind?: WorkflowFailureKind; + readonly failureCode?: WorkflowFailureCode; + readonly failureRecoverability?: WorkflowFailureRecoverability; + readonly failureDisposition?: WorkflowFailureDisposition; readonly failureMessage?: string; readonly failedStageId?: string; readonly resumable?: boolean; + readonly retryAfterMs?: number; +} + +export interface RunBlockedMetadata extends RunEndMetadata { + readonly failureRecoverability: "recoverable"; + readonly failedStageId: string; + readonly resumable: true; + readonly blockedAt?: number; +} + +function clearRunFailureMetadata(run: RunSnapshot): void { + delete run.error; + delete run.failureKind; + delete run.failureCode; + delete run.failureRecoverability; + delete run.failureDisposition; + delete run.failureMessage; + delete run.failedStageId; + delete run.resumable; + delete run.retryAfterMs; + delete run.blockedAt; +} + +function clearStaleBlockedRunMetadata(run: RunSnapshot, metadata: RunEndMetadata | undefined): void { + if (metadata?.failureKind === undefined) delete run.failureKind; + if (metadata?.failureCode === undefined) delete run.failureCode; + if (metadata?.failureRecoverability === undefined) delete run.failureRecoverability; + if (metadata?.failureDisposition === undefined) delete run.failureDisposition; + if (metadata?.failureMessage === undefined) delete run.failureMessage; + if (metadata?.failedStageId === undefined) delete run.failedStageId; + if (metadata?.resumable === undefined) delete run.resumable; + if (metadata?.retryAfterMs === undefined) delete run.retryAfterMs; +} + +function applyRunEndMetadata(run: RunSnapshot, metadata: RunEndMetadata): void { + if (metadata.failureKind !== undefined) run.failureKind = metadata.failureKind; + if (metadata.failureCode !== undefined) run.failureCode = metadata.failureCode; + if (metadata.failureRecoverability !== undefined) run.failureRecoverability = metadata.failureRecoverability; + if (metadata.failureDisposition !== undefined) run.failureDisposition = metadata.failureDisposition; + if (metadata.retryAfterMs !== undefined) run.retryAfterMs = metadata.retryAfterMs; + if (metadata.failureMessage !== undefined) run.failureMessage = metadata.failureMessage; + if (metadata.failedStageId !== undefined) run.failedStageId = metadata.failedStageId; + if (metadata.resumable !== undefined) run.resumable = metadata.resumable; } export type StagePromptAnswerSource = "workflow_ui" | "workflow_tool"; @@ -95,6 +144,12 @@ export interface Store { error?: string, metadata?: RunEndMetadata, ): boolean; + /** + * Record an active, recoverable workflow failure without ending the run. + * The run remains resumable/running and carries failure metadata for status, + * persistence restore, and continuation decisions. + */ + recordRunBlocked(runId: string, error: string, metadata: RunBlockedMetadata): boolean; /** * Remove a run from live workflow history/status. Any pending HIL prompt * waiter is rejected because the workflow will not resume through that path. @@ -448,6 +503,10 @@ export function createStore(): Store { existing.result = stage.result; existing.error = stage.error; existing.failureKind = stage.failureKind; + existing.failureCode = stage.failureCode; + existing.failureRecoverability = stage.failureRecoverability; + existing.failureDisposition = stage.failureDisposition; + existing.retryAfterMs = stage.retryAfterMs; existing.failureMessage = stage.failureMessage; existing.skippedReason = stage.skippedReason; if (stage.replayKey !== undefined) existing.replayKey = stage.replayKey; @@ -485,17 +544,26 @@ export function createStore(): Store { run.pausedAt = undefined; } run.durationMs = elapsedRunMs(run, run.endedAt); - if (status === "completed" && result !== undefined) { - run.result = result; - } - if ((status === "failed" || status === "killed") && error !== undefined) { - run.error = error; - } - if (metadata !== undefined) { - if (metadata.failureKind !== undefined) run.failureKind = metadata.failureKind; - if (metadata.failureMessage !== undefined) run.failureMessage = metadata.failureMessage; - if (metadata.failedStageId !== undefined) run.failedStageId = metadata.failedStageId; - if (metadata.resumable !== undefined) run.resumable = metadata.resumable; + const wasBlocked = run.blockedAt !== undefined || run.failureDisposition === "active_blocked"; + delete run.blockedAt; + if (status === "completed") { + if (result !== undefined) { + run.result = result; + } + clearRunFailureMetadata(run); + } else { + if (wasBlocked && error === undefined) delete run.error; + if ((status === "failed" || status === "killed") && error !== undefined) { + run.error = error; + } + if (wasBlocked) clearStaleBlockedRunMetadata(run, metadata); + if (metadata !== undefined) applyRunEndMetadata(run, metadata); + if (run.failureDisposition === "active_blocked") delete run.failureDisposition; + if (status === "killed") { + run.failureRecoverability = "non_recoverable"; + run.failureDisposition = "terminal_killed"; + run.resumable = false; + } } // Abandon any waiting HIL prompt — workflow body never resumed past // it, but the awaiter promise must reject so the executor's catch @@ -511,6 +579,26 @@ export function createStore(): Store { return true; }, + recordRunBlocked(runId: string, error: string, metadata: RunBlockedMetadata): boolean { + const run = findRun(runId); + if (!run) return false; + if (TERMINAL_STATUSES.has(run.status)) return false; + run.status = "running"; + run.error = error; + run.failureKind = metadata.failureKind; + run.failureCode = metadata.failureCode; + run.failureRecoverability = metadata.failureRecoverability; + run.failureDisposition = metadata.failureDisposition; + run.failureMessage = metadata.failureMessage; + run.failedStageId = metadata.failedStageId; + run.resumable = metadata.resumable; + run.blockedAt = metadata.blockedAt ?? Date.now(); + if (metadata.retryAfterMs !== undefined) run.retryAfterMs = metadata.retryAfterMs; + _version++; + notify(); + return true; + }, + removeRun(runId: string): boolean { const index = _runs.findIndex((r) => r.id === runId); if (index < 0) return false; diff --git a/packages/workflows/src/shared/workflow-failures.ts b/packages/workflows/src/shared/workflow-failures.ts index 5bc7ffcd5..4d4a4c852 100644 --- a/packages/workflows/src/shared/workflow-failures.ts +++ b/packages/workflows/src/shared/workflow-failures.ts @@ -1,19 +1,41 @@ -import type { WorkflowFailureKind } from "./store-types.js"; +import type { + WorkflowFailureCode, + WorkflowFailureDisposition, + WorkflowFailureKind, + WorkflowFailureRecoverability, +} from "./store-types.js"; export interface WorkflowFailure { readonly kind: WorkflowFailureKind; - /** Original error text, preserved for diagnostics. */ + /** Specific additive reason within the existing broad failure kind. */ + readonly code?: WorkflowFailureCode; + /** Redacted diagnostic text safe for snapshots and persistence. */ readonly message: string; /** Sanitized workflow-facing text shown on run/stage snapshots. */ readonly userMessage: string; readonly retryable: boolean; readonly resumable: boolean; + readonly recoverability: WorkflowFailureRecoverability; + readonly disposition: WorkflowFailureDisposition; + readonly retryAfterMs?: number; readonly cause?: unknown; } export const WORKFLOW_AUTH_FAILURE_MESSAGE = "You must be logged in to run workflows. Run /login and try again."; +export const WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE = + "A required model provider API key is missing. Configure the provider credentials and resume the workflow."; + +export const WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE = + "The configured model provider credentials are invalid. Update the provider API key, then start a new workflow run."; + +export const WORKFLOW_FORBIDDEN_MODEL_CONFIG_MESSAGE = + "The configured model provider or model is not available with the current credentials. Update the model configuration, then start a new workflow run."; + +export const WORKFLOW_UNKNOWN_MODEL_MESSAGE = + "The configured model is not available. Update the workflow model configuration, then start a new workflow run."; + const WORKFLOW_FAILURE_KINDS: ReadonlySet = new Set([ "auth", "rate_limit", @@ -26,22 +48,56 @@ export function isWorkflowFailureKind(kind: string): kind is WorkflowFailureKind return WORKFLOW_FAILURE_KINDS.has(kind as WorkflowFailureKind); } +export function isWorkflowFailureCode(code: string): code is WorkflowFailureCode { + switch (code) { + case "login_required": + case "missing_api_key": + case "invalid_api_key": + case "forbidden_config": + case "unknown_model": + case "rate_limited": + case "quota_limited": + case "provider_unavailable": + case "cancelled": + case "unknown": + return true; + default: + return false; + } +} + +export function isWorkflowFailureRecoverability(value: string): value is WorkflowFailureRecoverability { + return value === "recoverable" || value === "non_recoverable" || value === "unknown"; +} + +export function isWorkflowFailureDisposition(value: string): value is WorkflowFailureDisposition { + return value === "active_blocked" || value === "terminal_killed" || value === "terminal_failed"; +} + function makeWorkflowFailure( kind: WorkflowFailureKind, message: string, opts: { readonly retryable: boolean; readonly resumable: boolean; + readonly recoverability: WorkflowFailureRecoverability; + readonly disposition: WorkflowFailureDisposition; readonly cause: unknown; + readonly code?: WorkflowFailureCode; + readonly retryAfterMs?: number; readonly userMessage?: string; }, ): WorkflowFailure { return { kind, + ...(opts.code !== undefined ? { code: opts.code } : {}), message, - userMessage: opts.userMessage ?? message, + userMessage: opts.userMessage ?? redactSensitiveText(message), retryable: opts.retryable, resumable: opts.resumable, + recoverability: opts.recoverability, + disposition: opts.disposition, + ...(opts.retryAfterMs !== undefined ? { retryAfterMs: opts.retryAfterMs } : {}), cause: opts.cause, }; } @@ -82,6 +138,35 @@ type StructuredSignal = { readonly code?: string | number; readonly name?: string; readonly stopReason?: string; + readonly message?: string; + readonly retryAfterMs?: number; +}; + +type WorkflowFailureDecision = { + readonly kind: WorkflowFailureKind; + readonly code: WorkflowFailureCode; + readonly retryable: boolean; + readonly resumable: boolean; + readonly recoverability: WorkflowFailureRecoverability; + readonly disposition: WorkflowFailureDisposition; + readonly userMessage?: string; + readonly retryAfterMs?: number; +}; + +type WorkflowFailureClassificationSource = + | "top_level" + | "diagnostic" + | "nested" + | "cause" + | "aggregate"; + +type WorkflowFailureEvidence = "strong_signal" | "weak_signal" | "message" | "status"; + +type WorkflowFailureClassification = { + readonly decision: WorkflowFailureDecision; + readonly source: WorkflowFailureClassificationSource; + readonly evidence: WorkflowFailureEvidence; + readonly message?: string; }; function integerFrom(value: unknown): number | undefined { @@ -91,17 +176,61 @@ function integerFrom(value: unknown): number | undefined { return Number.isInteger(parsed) ? parsed : undefined; } +function numberFrom(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string" || value.trim().length === 0) return undefined; + const parsed = Number(value.trim()); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function retryAfterHeaderMs(value: unknown): number | undefined { + const numeric = numberFrom(value); + if (numeric !== undefined && numeric >= 0) return Math.round(numeric * 1000); + if (typeof value !== "string" || value.trim().length === 0) return undefined; + const dateMs = Date.parse(value); + if (!Number.isFinite(dateMs)) return undefined; + return Math.max(0, dateMs - Date.now()); +} + +function retryAfterMsFrom(error: unknown): number | undefined { + const directMs = numberFrom(field(error, "retryAfterMs")); + if (directMs !== undefined && directMs >= 0) return Math.round(directMs); + + const seconds = numberFrom(field(error, "retryAfterSeconds")); + if (seconds !== undefined && seconds >= 0) return Math.round(seconds * 1000); + + // Provider SDKs commonly mirror the HTTP Retry-After header as retryAfter, + // so the ambiguous bare field follows header semantics (seconds/date). Use + // retryAfterMs for explicit millisecond values. + const retryAfter = retryAfterHeaderMs(field(error, "retryAfter")); + if (retryAfter !== undefined) return retryAfter; + + const retryAfterHeader = retryAfterHeaderMs(field(error, "retry-after")); + if (retryAfterHeader !== undefined) return retryAfterHeader; + + const headers = field(error, "headers"); + const headerRecord = asRecord(headers); + const headerValue = headerRecord?.["retry-after"] ?? headerRecord?.["Retry-After"]; + return retryAfterHeaderMs(headerValue); +} + function structuredSignal(error: unknown): StructuredSignal { const status = integerFrom(field(error, "status")) ?? integerFrom(field(error, "statusCode")) ?? integerFrom(field(error, "httpStatus")); const rawCode = field(error, "code"); const code = typeof rawCode === "string" || typeof rawCode === "number" ? rawCode : undefined; + const name = errorName(error); + const stopReason = stringField(error, "stopReason"); + const message = structuredErrorMessage(error); + const retryAfterMs = retryAfterMsFrom(error); return { ...(status !== undefined ? { status } : {}), ...(code !== undefined ? { code } : {}), - ...(errorName(error) !== undefined ? { name: errorName(error)! } : {}), - ...(stringField(error, "stopReason") !== undefined ? { stopReason: stringField(error, "stopReason")! } : {}), + ...(name !== undefined ? { name } : {}), + ...(stopReason !== undefined ? { stopReason } : {}), + ...(message !== undefined ? { message } : {}), + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), }; } @@ -121,121 +250,38 @@ function diagnosticErrors(error: unknown): readonly unknown[] { return errors; } +function nestedProviderError(error: unknown): unknown { + return field(error, "error") ?? field(error, "response") ?? field(error, "body"); +} + function normalizeCode(value: string | number | undefined): string | undefined { if (value === undefined) return undefined; - return String(value).trim().toLowerCase().replaceAll("-", "_"); + return String(value).trim().toLowerCase().replaceAll("-", "_").replaceAll(" ", "_"); } -function kindFromStatus(status: number | undefined): WorkflowFailureKind | undefined { - switch (status) { - case 401: - case 403: - return "auth"; - case 429: - return "rate_limit"; - case 500: - case 502: - case 503: - case 504: - return "provider"; - default: - return undefined; - } -} +type StructuredCodeEvidence = + | { readonly kind: "semantic_code"; readonly normalized: string } + | { readonly kind: "wrapper_http_status"; readonly status: number }; -function kindFromCode(code: string | number | undefined): WorkflowFailureKind | undefined { - const normalized = normalizeCode(code); - switch (normalized) { - case undefined: - return undefined; - case "401": - case "403": - case "auth": - case "auth_required": - case "authentication_required": - case "unauthorized": - case "forbidden": - case "invalid_api_key": - case "missing_api_key": - return "auth"; - case "429": - case "rate_limit": - case "rate_limit_exceeded": - case "too_many_requests": - case "quota_exceeded": - return "rate_limit"; - case "aborterror": - case "aborted": - case "cancelled": - case "canceled": - return "cancelled"; - case "500": - case "502": - case "503": - case "504": - case "provider_error": - case "service_unavailable": - case "temporarily_unavailable": - case "overloaded": - return "provider"; - default: - return undefined; +function httpStatusFromCode(value: string | number | undefined): number | undefined { + if (value === undefined) return undefined; + if (typeof value === "number") { + return Number.isInteger(value) && value >= 100 && value <= 599 ? value : undefined; } + const trimmed = value.trim(); + if (!/^\d{3}$/.test(trimmed)) return undefined; + const parsed = Number(trimmed); + return parsed >= 100 && parsed <= 599 ? parsed : undefined; } -function structuredKind(error: unknown, seen = new Set()): WorkflowFailureKind | undefined { - if (error === undefined || error === null || seen.has(error)) return undefined; - if (typeof error === "object") seen.add(error); - - const signal = structuredSignal(error); - if (signal.stopReason?.toLowerCase() === "aborted") return "cancelled"; - const statusKind = kindFromStatus(signal.status); - if (statusKind !== undefined) return statusKind; - const codeKind = kindFromCode(signal.code) ?? kindFromCode(signal.name); - if (codeKind !== undefined) return codeKind; - - for (const diagnosticError of diagnosticErrors(error)) { - const diagnosticKind = structuredKind(diagnosticError, seen); - if (diagnosticKind !== undefined) return diagnosticKind; - } +function codeEvidenceFrom(value: string | number | undefined): StructuredCodeEvidence | undefined { + const status = httpStatusFromCode(value); + if (status !== undefined) return { kind: "wrapper_http_status", status }; - return structuredKind(causeOf(error), seen); -} - -function failureForKind(kind: WorkflowFailureKind, message: string, cause: unknown): WorkflowFailure { - switch (kind) { - case "auth": - return makeWorkflowFailure("auth", message, { - userMessage: WORKFLOW_AUTH_FAILURE_MESSAGE, - retryable: true, - resumable: true, - cause, - }); - case "rate_limit": - return makeWorkflowFailure("rate_limit", message, { - retryable: true, - resumable: true, - cause, - }); - case "cancelled": - return makeWorkflowFailure("cancelled", message, { - retryable: false, - resumable: false, - cause, - }); - case "provider": - return makeWorkflowFailure("provider", message, { - retryable: true, - resumable: true, - cause, - }); - case "unknown": - return makeWorkflowFailure("unknown", message, { - retryable: false, - resumable: true, - cause, - }); - } + const normalized = normalizeCode(value); + return normalized !== undefined && normalized.length > 0 + ? { kind: "semantic_code", normalized } + : undefined; } type TokenMatch = readonly string[]; @@ -286,24 +332,141 @@ function tokenNearAny(tokens: readonly string[], anchor: string, candidates: Rea return false; } -const AUTH_PHRASES: readonly TokenMatch[] = [ - ["no", "api", "key"], - ["api", "key", "not", "found"], - ["missing", "api", "key"], - ["no", "model", "selected"], - ["no", "models", "available"], +const INVALID_API_KEY_CODES = new Set([ + "401", + "invalid_api_key", + "incorrect_api_key", + "invalid_api_key_error", + "invalid_credentials", + "bad_credentials", + "authentication_error", +]); + +const LOGIN_REQUIRED_CODES = new Set([ + "auth", + "auth_required", + "authentication_required", + "login_required", + "not_logged_in", +]); + +const MISSING_API_KEY_CODES = new Set([ + "missing_api_key", + "api_key_missing", + "no_api_key", +]); + +const RATE_LIMIT_CODES = new Set([ + "429", + "rate_limit", + "rate_limited", + "rate_limit_exceeded", + "too_many_requests", +]); + +const QUOTA_LIMIT_CODES = new Set([ + "quota", + "quota_exceeded", + "insufficient_quota", + "usage_limit", + "usage_limit_exceeded", +]); + +const CANCELLED_CODES = new Set([ + "aborterror", + "aborted", + "cancelled", + "canceled", +]); + +const PROVIDER_UNAVAILABLE_CODES = new Set([ + "500", + "502", + "503", + "504", + "provider_error", + "service_unavailable", + "temporarily_unavailable", + "overloaded", + "timeout", + "network_error", +]); + +const UNKNOWN_MODEL_CODES = new Set([ + "unknown_model", + "model_not_found", + "model_not_available", + "unsupported_model", +]); + +const FORBIDDEN_CONFIG_CODES = new Set([ + "403", + "forbidden", + "permission_denied", + "access_denied", + "forbidden_config", + "invalid_model_config", + "model_access_denied", +]); + +const LOGIN_REQUIRED_PHRASES: readonly TokenMatch[] = [ ["not", "logged", "in"], ["log", "in"], ["login", "required"], ["authentication", "required"], + ["please", "login"], + ["please", "log", "in"], ["unauthorized"], ]; +const LOCAL_LOGIN_REQUIRED_PHRASES: readonly TokenMatch[] = [ + ["not", "logged", "in"], + ["login", "required"], + ["please", "login"], + ["please", "log", "in"], + ["log", "in", "to", "continue"], +]; + +const PROVIDER_AUTH_FALLBACK_PHRASES: readonly TokenMatch[] = [ + ["unauthorized"], + ["authentication", "required"], +]; + +const MISSING_API_KEY_PHRASES: readonly TokenMatch[] = [ + ["no", "api", "key"], + ["api", "key", "not", "found"], + ["missing", "api", "key"], + ["api", "key", "missing"], + ["no", "model", "selected"], + ["no", "models", "available"], +]; + +const INVALID_API_KEY_PHRASES: readonly TokenMatch[] = [ + ["incorrect", "api", "key"], + ["invalid", "api", "key"], + ["api", "key", "invalid"], + ["api", "key", "incorrect"], + ["invalid", "credentials"], + ["invalid", "credential"], +]; + +const INVALID_API_KEY_CONTEXT = new Set(["invalid", "incorrect"]); + +const HTTP_RATE_LIMIT_PHRASES: readonly TokenMatch[] = [ + ["429"], + ["too", "many", "requests"], +]; + const RATE_LIMIT_PHRASES: readonly TokenMatch[] = [ ["rate", "limit"], - ["429"], + ["rate", "limited"], +]; + +const QUOTA_LIMIT_PHRASES: readonly TokenMatch[] = [ ["quota"], - ["too", "many", "requests"], + ["quota", "exceeded"], + ["insufficient", "quota"], + ["usage", "limit"], ]; const CANCELLED_PHRASES: readonly TokenMatch[] = [ @@ -312,11 +475,30 @@ const CANCELLED_PHRASES: readonly TokenMatch[] = [ ["canceled"], ]; -const PROVIDER_PHRASES: readonly TokenMatch[] = [ +const UNKNOWN_MODEL_PHRASES: readonly TokenMatch[] = [ ["model", "not", "found"], + ["unknown", "model"], + ["unsupported", "model"], + ["model", "does", "not", "exist"], + ["model", "not", "available"], +]; + +const FORBIDDEN_CONFIG_PHRASES: readonly TokenMatch[] = [ + ["forbidden", "config"], + ["forbidden", "configuration"], + ["permission", "denied"], + ["access", "denied"], + ["not", "allowed", "to", "access", "model"], + ["does", "not", "have", "access", "to", "model"], +]; + +const PROVIDER_UNAVAILABLE_PHRASES: readonly TokenMatch[] = [ ["overloaded"], ["temporarily", "unavailable"], ["service", "unavailable"], + ["provider", "unavailable"], + ["provider", "error"], + ["model", "unavailable"], ["503"], ]; @@ -350,26 +532,470 @@ const PROVIDER_CONTEXT = new Set([ "service", ]); -function fallbackKindFromMessage(message: string, name: string | undefined): WorkflowFailureKind | undefined { - const tokens = tokenize(message); - if (hasAnyPhrase(tokens, AUTH_PHRASES) || tokenNearAny(tokens, "oauth", AUTH_CONTEXT, 8)) return "auth"; - if (hasAnyPhrase(tokens, RATE_LIMIT_PHRASES)) return "rate_limit"; - if (name?.toLowerCase() === "aborterror" || hasAnyPhrase(tokens, CANCELLED_PHRASES)) return "cancelled"; +function redactedSecretReplacement(prefix: string): string { + return `${prefix}[redacted]`; +} + +function redactSensitiveText(value: string): string { + return value + .replace(/(sk-[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+/g, redactedSecretReplacement("$1")) + .replace(/\b(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[redacted]") + .replace(/\b(bearer\s+)[A-Za-z0-9._~+/-]{8,}=*/gi, "$1[redacted]") + .replace(/((?:api[_-]?key|token|credential|secret)\s*[:=]\s*)[^\s,;]+/gi, "$1[redacted]"); +} + +function authDecision(code: WorkflowFailureCode): WorkflowFailureDecision { + if (code === "invalid_api_key") { + return { + kind: "auth", + code, + retryable: false, + resumable: false, + recoverability: "non_recoverable", + disposition: "terminal_killed", + userMessage: WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE, + }; + } + if (code === "missing_api_key") { + return { + kind: "auth", + code, + retryable: true, + resumable: true, + recoverability: "recoverable", + disposition: "active_blocked", + userMessage: WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE, + }; + } + return { + kind: "auth", + code: "login_required", + retryable: true, + resumable: true, + recoverability: "recoverable", + disposition: "active_blocked", + userMessage: WORKFLOW_AUTH_FAILURE_MESSAGE, + }; +} + +function rateLimitDecision( + code: "rate_limited" | "quota_limited", + retryAfterMs?: number, +): WorkflowFailureDecision { + return { + kind: "rate_limit", + code, + retryable: true, + resumable: true, + recoverability: "recoverable", + disposition: "active_blocked", + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }; +} + +function providerUnavailableDecision(retryAfterMs?: number): WorkflowFailureDecision { + return { + kind: "provider", + code: "provider_unavailable", + retryable: true, + resumable: true, + recoverability: "recoverable", + disposition: "active_blocked", + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }; +} + +function terminalProviderConfigDecision(code: "forbidden_config" | "unknown_model"): WorkflowFailureDecision { + return { + kind: "provider", + code, + retryable: false, + resumable: false, + recoverability: "non_recoverable", + disposition: "terminal_killed", + userMessage: code === "unknown_model" ? WORKFLOW_UNKNOWN_MODEL_MESSAGE : WORKFLOW_FORBIDDEN_MODEL_CONFIG_MESSAGE, + }; +} + +function cancelledDecision(): WorkflowFailureDecision { + return { + kind: "cancelled", + code: "cancelled", + retryable: false, + resumable: false, + recoverability: "non_recoverable", + disposition: "terminal_killed", + }; +} + +function unknownDecision(): WorkflowFailureDecision { + return { + kind: "unknown", + code: "unknown", + retryable: false, + resumable: true, + recoverability: "unknown", + disposition: "terminal_failed", + }; +} + +function strongDecisionFromNormalizedCode(normalized: string | undefined, retryAfterMs?: number): WorkflowFailureDecision | undefined { + if (normalized === undefined) return undefined; + if (CANCELLED_CODES.has(normalized)) return cancelledDecision(); + if (INVALID_API_KEY_CODES.has(normalized)) return authDecision("invalid_api_key"); + if (MISSING_API_KEY_CODES.has(normalized)) return authDecision("missing_api_key"); + if (RATE_LIMIT_CODES.has(normalized)) return rateLimitDecision("rate_limited", retryAfterMs); + if (QUOTA_LIMIT_CODES.has(normalized)) return rateLimitDecision("quota_limited", retryAfterMs); + if (UNKNOWN_MODEL_CODES.has(normalized)) return terminalProviderConfigDecision("unknown_model"); + if (FORBIDDEN_CONFIG_CODES.has(normalized)) return terminalProviderConfigDecision("forbidden_config"); + if (PROVIDER_UNAVAILABLE_CODES.has(normalized)) return providerUnavailableDecision(retryAfterMs); + return undefined; +} + +function weakLoginDecisionFromNormalizedCode(normalized: string | undefined): WorkflowFailureDecision | undefined { + return normalized !== undefined && LOGIN_REQUIRED_CODES.has(normalized) + ? authDecision("login_required") + : undefined; +} + +function classificationForDecision( + decision: WorkflowFailureDecision, + source: WorkflowFailureClassificationSource, + message: string | undefined, + evidence: WorkflowFailureEvidence = "message", +): WorkflowFailureClassification { + return { + decision, + source, + evidence, + ...(message !== undefined ? { message } : {}), + }; +} + +function hasInvalidApiKeyMessage(tokens: readonly string[]): boolean { + return hasAnyPhrase(tokens, INVALID_API_KEY_PHRASES) + || (hasPhrase(tokens, ["api", "key"]) && tokenNearAny(tokens, "key", INVALID_API_KEY_CONTEXT, 6)); +} + +function decisionFromMessageTokens(tokens: readonly string[], name: string | undefined, retryAfterMs?: number): WorkflowFailureDecision | undefined { + if (name?.toLowerCase() === "aborterror" || hasAnyPhrase(tokens, CANCELLED_PHRASES)) return cancelledDecision(); + if (hasAnyPhrase(tokens, HTTP_RATE_LIMIT_PHRASES)) return rateLimitDecision("rate_limited", retryAfterMs); + if (hasAnyPhrase(tokens, QUOTA_LIMIT_PHRASES)) return rateLimitDecision("quota_limited", retryAfterMs); + if (hasAnyPhrase(tokens, RATE_LIMIT_PHRASES)) return rateLimitDecision("rate_limited", retryAfterMs); + if (hasInvalidApiKeyMessage(tokens)) return authDecision("invalid_api_key"); + if (hasAnyPhrase(tokens, MISSING_API_KEY_PHRASES)) return authDecision("missing_api_key"); + if (hasAnyPhrase(tokens, LOGIN_REQUIRED_PHRASES) || tokenNearAny(tokens, "oauth", AUTH_CONTEXT, 8)) return authDecision("login_required"); + if (hasAnyPhrase(tokens, UNKNOWN_MODEL_PHRASES)) return terminalProviderConfigDecision("unknown_model"); + if (hasAnyPhrase(tokens, FORBIDDEN_CONFIG_PHRASES)) return terminalProviderConfigDecision("forbidden_config"); if ( - hasAnyPhrase(tokens, PROVIDER_PHRASES) + hasAnyPhrase(tokens, PROVIDER_UNAVAILABLE_PHRASES) || tokenNearAny(tokens, "model", MODEL_PROVIDER_CONTEXT, 8) || tokenNearAny(tokens, "provider", PROVIDER_CONTEXT, 8) - ) return "provider"; + ) return providerUnavailableDecision(retryAfterMs); return undefined; } +function decisionFromStatus(status: number | undefined, retryAfterMs: number | undefined): WorkflowFailureDecision | undefined { + switch (status) { + case 401: + return authDecision("invalid_api_key"); + case 403: + return terminalProviderConfigDecision("forbidden_config"); + case 429: + return rateLimitDecision("rate_limited", retryAfterMs); + case 500: + case 502: + case 503: + case 504: + return providerUnavailableDecision(retryAfterMs); + default: + return undefined; + } +} + +const STATUS_MESSAGE_REFINEMENT_CODES: ReadonlySet = new Set([ + "invalid_api_key", + "missing_api_key", + "unknown_model", + "forbidden_config", +]); + +const BROAD_AUTH_MESSAGE_REFINEMENT_CODES: ReadonlySet = new Set([ + "invalid_api_key", + "missing_api_key", +]); + +const STATUS_RELATED_MESSAGE_REFINEMENT_CODES: ReadonlySet = new Set([ + "invalid_api_key", + "missing_api_key", + "unknown_model", + "forbidden_config", + "rate_limited", + "quota_limited", + "cancelled", +]); + +function isRecoverableActiveBlocked(classification: WorkflowFailureClassification): boolean { + return classification.decision.disposition === "active_blocked" + && classification.decision.recoverability === "recoverable"; +} + +function canUseRelatedClassificationBeforeStatus(classification: WorkflowFailureClassification): boolean { + if (classification.evidence === "weak_signal") return false; + if (classification.evidence === "message") { + return STATUS_RELATED_MESSAGE_REFINEMENT_CODES.has(classification.decision.code); + } + return classification.decision.code !== "login_required"; +} + +function isClearLocalLoginMessage(message: string, tokens: readonly string[] = tokenize(message)): boolean { + if (message.toLowerCase().includes("/login")) return true; + return hasAnyPhrase(tokens, LOCAL_LOGIN_REQUIRED_PHRASES); +} + +function hasFallbackApiError401(tokens: readonly string[]): boolean { + return hasPhrase(tokens, ["401"]) && hasPhrase(tokens, ["api", "error"]); +} + +function classifyFallbackProviderAuthMessage(message: string, tokens: readonly string[]): WorkflowFailureDecision | undefined { + if (isClearLocalLoginMessage(message, tokens)) return undefined; + return hasAnyPhrase(tokens, PROVIDER_AUTH_FALLBACK_PHRASES) || hasFallbackApiError401(tokens) + ? authDecision("invalid_api_key") + : undefined; +} + +function canUseLoginClassificationBeforeWrapper401( + classification: WorkflowFailureClassification | undefined, +): classification is WorkflowFailureClassification { + if (classification === undefined || classification.decision.code !== "login_required") return false; + return classification.evidence === "weak_signal" + || classification.evidence === "strong_signal" + || (classification.message !== undefined && isClearLocalLoginMessage(classification.message)); +} + +function classificationFromNormalizedCode( + normalized: string | undefined, + retryAfterMs: number | undefined, + source: WorkflowFailureClassificationSource, + message: string | undefined, +): { readonly strong?: WorkflowFailureClassification; readonly weak?: WorkflowFailureClassification } { + const strong = strongDecisionFromNormalizedCode(normalized, retryAfterMs); + if (strong !== undefined) { + return { strong: classificationForDecision(strong, source, message, "strong_signal") }; + } + const weak = weakLoginDecisionFromNormalizedCode(normalized); + return weak !== undefined + ? { weak: classificationForDecision(weak, source, message, "weak_signal") } + : {}; +} + +function aggregateErrorItems(error: unknown): readonly unknown[] { + const nativeErrors = error instanceof AggregateError ? error.errors as unknown : undefined; + const errors = nativeErrors ?? field(error, "errors"); + return Array.isArray(errors) ? errors : []; +} + +function fallbackAggregateClassification(innerError: unknown): WorkflowFailureClassification { + const message = errorMessage(innerError); + const fallback = fallbackDecisionFromMessage(message, errorName(innerError)); + return classificationForDecision(fallback ?? unknownDecision(), "aggregate", message); +} + +function recoverableBlockedClassification(classifications: readonly WorkflowFailureClassification[]): WorkflowFailureClassification { + return classifications.find((classification) => classification.decision.retryAfterMs !== undefined) + ?? classifications[0]!; +} + +function aggregateClassification(error: unknown, seen: Set): WorkflowFailureClassification | undefined { + const innerErrors = aggregateErrorItems(error); + if (innerErrors.length === 0) return undefined; + + const classifications = innerErrors.map((innerError) => { + const branchSeen = new Set(seen); + return structuredClassification(innerError, "aggregate", branchSeen) ?? fallbackAggregateClassification(innerError); + }); + + const terminalKilled = classifications.find( + (classification) => classification.decision.disposition === "terminal_killed", + ); + if (terminalKilled !== undefined) return terminalKilled; + + const allRecoverableBlocked = classifications.every(isRecoverableActiveBlocked); + if (allRecoverableBlocked) return recoverableBlockedClassification(classifications); + + return classificationForDecision(unknownDecision(), "aggregate", errorMessage(error)); +} + +function selectDiagnosticFailureClassification( + diagnostics: readonly unknown[], + seen: ReadonlySet, +): WorkflowFailureClassification | undefined { + const classifications: WorkflowFailureClassification[] = []; + for (const diagnosticError of diagnostics) { + const diagnosticSeen = new Set(seen); + const diagnosticClassification = structuredClassification(diagnosticError, "diagnostic", diagnosticSeen); + if (diagnosticClassification !== undefined) classifications.push(diagnosticClassification); + } + if (classifications.length === 0) return undefined; + + const terminalKilled = classifications.find( + (classification) => classification.decision.disposition === "terminal_killed", + ); + if (terminalKilled !== undefined) return terminalKilled; + + const terminalFailed = classifications.find( + (classification) => classification.decision.disposition === "terminal_failed", + ); + if (terminalFailed !== undefined) return terminalFailed; + + const allRecoverableBlocked = classifications.every(isRecoverableActiveBlocked); + if (allRecoverableBlocked) return recoverableBlockedClassification(classifications); + + return classifications[0]!; +} + +function relatedStructuredClassification(error: unknown, seen: Set): WorkflowFailureClassification | undefined { + const diagnosticClassification = selectDiagnosticFailureClassification(diagnosticErrors(error), seen); + if (diagnosticClassification !== undefined) return diagnosticClassification; + + const nested = nestedProviderError(error); + if (nested !== undefined && nested !== error) { + const nestedClassification = structuredClassification(nested, "nested", seen); + if (nestedClassification !== undefined) return nestedClassification; + } + + const causeClassification = structuredClassification(causeOf(error), "cause", seen); + if (causeClassification !== undefined) return causeClassification; + + return aggregateClassification(error, seen); +} + +function structuredClassification( + error: unknown, + source: WorkflowFailureClassificationSource = "top_level", + seen = new Set(), +): WorkflowFailureClassification | undefined { + if (error === undefined || error === null || seen.has(error)) return undefined; + if (typeof error === "object") seen.add(error); + + const signal = structuredSignal(error); + const signalMessage = signal.message ?? (typeof error === "string" ? error : undefined); + if (signal.stopReason?.toLowerCase() === "aborted") { + return classificationForDecision(cancelledDecision(), source, signalMessage, "strong_signal"); + } + + const retryAfterMs = signal.retryAfterMs; + let weakClassification: WorkflowFailureClassification | undefined; + + const codeEvidence = codeEvidenceFrom(signal.code); + if (codeEvidence?.kind === "semantic_code") { + const codeClassification = classificationFromNormalizedCode(codeEvidence.normalized, retryAfterMs, source, signalMessage); + if (codeClassification.strong !== undefined) return codeClassification.strong; + weakClassification = codeClassification.weak ?? weakClassification; + } + + const nameClassification = classificationFromNormalizedCode(normalizeCode(signal.name), retryAfterMs, source, signalMessage); + if (nameClassification.strong !== undefined) return nameClassification.strong; + weakClassification = nameClassification.weak ?? weakClassification; + + const messageTokens = signalMessage !== undefined ? tokenize(signalMessage) : undefined; + const messageDecision = messageTokens !== undefined + ? decisionFromMessageTokens(messageTokens, signal.name, retryAfterMs) + : undefined; + const providerAuthMessageDecision = signalMessage !== undefined && messageTokens !== undefined + ? classifyFallbackProviderAuthMessage(signalMessage, messageTokens) + : undefined; + if ( + weakClassification !== undefined && + messageDecision !== undefined && + BROAD_AUTH_MESSAGE_REFINEMENT_CODES.has(messageDecision.code) + ) { + return classificationForDecision(messageDecision, source, signalMessage); + } + + const relatedClassification = relatedStructuredClassification(error, seen); + const effectiveStatus = signal.status ?? (codeEvidence?.kind === "wrapper_http_status" ? codeEvidence.status : undefined); + const statusDecision = decisionFromStatus(effectiveStatus, retryAfterMs); + if (statusDecision !== undefined) { + if (relatedClassification !== undefined && canUseRelatedClassificationBeforeStatus(relatedClassification)) { + return relatedClassification; + } + if ( + signalMessage !== undefined && + (effectiveStatus === 401 || effectiveStatus === 403) && + messageDecision !== undefined && + STATUS_MESSAGE_REFINEMENT_CODES.has(messageDecision.code) + ) { + return classificationForDecision(messageDecision, source, signalMessage); + } + if (effectiveStatus === 401) { + if (canUseLoginClassificationBeforeWrapper401(relatedClassification)) { + return relatedClassification; + } + if (canUseLoginClassificationBeforeWrapper401(weakClassification)) { + return weakClassification; + } + if (signalMessage !== undefined && isClearLocalLoginMessage(signalMessage)) { + return classificationForDecision(authDecision("login_required"), source, signalMessage); + } + } + return classificationForDecision(statusDecision, source, signalMessage, "status"); + } + + if (source !== "top_level") { + if ( + providerAuthMessageDecision !== undefined && + (messageDecision === undefined || messageDecision.code === "login_required") + ) { + return classificationForDecision(providerAuthMessageDecision, source, signalMessage); + } + if (messageDecision !== undefined) { + return classificationForDecision(messageDecision, source, signalMessage); + } + } + + if (relatedClassification !== undefined) return relatedClassification; + + return weakClassification; +} + +function fallbackDecisionFromMessage(message: string, name: string | undefined): WorkflowFailureDecision | undefined { + const tokens = tokenize(message); + const decision = decisionFromMessageTokens(tokens, name); + const providerAuthDecision = classifyFallbackProviderAuthMessage(message, tokens); + if (providerAuthDecision !== undefined && (decision === undefined || decision.code === "login_required")) { + return providerAuthDecision; + } + if (decision === undefined && isClearLocalLoginMessage(message, tokens)) { + return authDecision("login_required"); + } + return decision; +} + +function failureForDecision(decision: WorkflowFailureDecision, message: string, cause: unknown): WorkflowFailure { + const safeMessage = redactSensitiveText(message); + return makeWorkflowFailure(decision.kind, safeMessage, { + code: decision.code, + retryable: decision.retryable, + resumable: decision.resumable, + recoverability: decision.recoverability, + disposition: decision.disposition, + cause, + ...(decision.userMessage !== undefined ? { userMessage: decision.userMessage } : {}), + ...(decision.retryAfterMs !== undefined ? { retryAfterMs: decision.retryAfterMs } : {}), + }); +} + export function classifyWorkflowFailure(error: unknown): WorkflowFailure { const message = errorMessage(error); - const structured = structuredKind(error); - if (structured !== undefined) return failureForKind(structured, message, error); + const structured = structuredClassification(error); + if (structured !== undefined) { + const structuredMessage = structured.message ?? message; + return failureForDecision(structured.decision, structuredMessage, error); + } - const fallback = fallbackKindFromMessage(message, errorName(error)); - if (fallback !== undefined) return failureForKind(fallback, message, error); + const fallback = fallbackDecisionFromMessage(message, errorName(error)); + if (fallback !== undefined) return failureForDecision(fallback, message, error); - return failureForKind("unknown", message, error); + return failureForDecision(unknownDecision(), message, error); } diff --git a/test/manual/render-preview.ts b/test/manual/render-preview.ts index 8feea9dbf..cc4601fe3 100644 --- a/test/manual/render-preview.ts +++ b/test/manual/render-preview.ts @@ -202,6 +202,7 @@ const store: Store = { recordStageInputRequest: () => false, clearStageInputRequest: () => false, recordRunEnd: () => false, + recordRunBlocked: () => false, removeRun: () => false, recordNotice: () => {}, ackNotice: () => false, diff --git a/test/unit/background-status-kill.test.ts b/test/unit/background-status-kill.test.ts index 5ee247fb8..7d3dc99ae 100644 --- a/test/unit/background-status-kill.test.ts +++ b/test/unit/background-status-kill.test.ts @@ -6,7 +6,7 @@ import { describe, test } from "bun:test"; import assert from "node:assert/strict"; -import { killRun, killAllRuns, resumeRun } from "../../packages/workflows/src/runs/background/status.js"; +import { killRun, killAllRuns, resumeRun, inspectRun } from "../../packages/workflows/src/runs/background/status.js"; import { createStore } from "../../packages/workflows/src/shared/store.js"; import { createCancellationRegistry } from "../../packages/workflows/src/runs/background/cancellation-registry.js"; import type { WorkflowPersistencePort } from "../../packages/workflows/src/shared/types.js"; @@ -120,6 +120,78 @@ describe("killRun — with persistence", () => { }); }); +// --------------------------------------------------------------------------- +// killRun — active-blocked metadata cleanup +// --------------------------------------------------------------------------- + +describe("killRun — active-blocked metadata cleanup", () => { + test("terminalizes a blocked run as non-resumable killed and persists terminal metadata", () => { + const s = createStore(); + const { port, calls } = makePersistence(); + const run = makeRun({ id: "blocked-run" }); + s.recordRunStart(run); + s.recordRunBlocked(run.id, "rate limit exceeded", { + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "rate limit exceeded", + failedStageId: "limited-stage", + resumable: true, + retryAfterMs: 1000, + blockedAt: 1234, + }); + + const result = killRun(run.id, { store: s, persistence: port }); + + assert.deepEqual(result, { ok: true, runId: run.id, previousStatus: "running" }); + const stored = s.runs().find((r) => r.id === run.id); + assert.equal(stored?.status, "killed"); + assert.equal(typeof stored?.endedAt, "number"); + assert.equal(stored?.blockedAt, undefined); + assert.equal(stored?.resumable, false); + assert.equal(stored?.failureKind, "cancelled"); + assert.equal(stored?.failureCode, "cancelled"); + assert.equal(stored?.failureRecoverability, "non_recoverable"); + assert.equal(stored?.failureDisposition, "terminal_killed"); + assert.equal(stored?.failureMessage, "workflow killed"); + assert.equal(stored?.failedStageId, undefined); + assert.equal(stored?.retryAfterMs, undefined); + + const inspected = inspectRun(run.id, { store: s }); + assert.equal(inspected.ok, true); + if (!inspected.ok) throw new Error("narrowing"); + assert.equal(inspected.detail.status, "killed"); + assert.equal(inspected.detail.blockedAt, undefined); + assert.equal(inspected.detail.resumable, false); + assert.equal(inspected.detail.failureRecoverability, "non_recoverable"); + assert.equal(inspected.detail.failureDisposition, "terminal_killed"); + + const resumed = resumeRun(run.id, { store: s }); + assert.equal(resumed.ok, true); + if (!resumed.ok) throw new Error("narrowing"); + assert.equal(resumed.mode, "not_resumable"); + assert.equal(resumed.snapshot.blockedAt, undefined); + assert.equal(resumed.snapshot.resumable, false); + assert.equal(resumed.snapshot.failureDisposition, "terminal_killed"); + + assert.equal(calls.length, 1); + const payload = calls[0]?.payload; + assert.ok(payload); + assert.equal(payload.status, "killed"); + assert.equal(payload.runId, run.id); + assert.equal(payload.error, "workflow killed"); + assert.equal(payload.failureKind, "cancelled"); + assert.equal(payload.failureCode, "cancelled"); + assert.equal(payload.failureRecoverability, "non_recoverable"); + assert.equal(payload.failureDisposition, "terminal_killed"); + assert.equal(payload.failureMessage, "workflow killed"); + assert.equal(payload.resumable, false); + assert.equal(payload.failedStageId, undefined); + assert.equal(payload.retryAfterMs, undefined); + }); +}); + // --------------------------------------------------------------------------- // killRun — abort wiring (cancellation checked AFTER run validation) // --------------------------------------------------------------------------- diff --git a/test/unit/background-status.test.ts b/test/unit/background-status.test.ts index 6e3672096..ec67f1828 100644 --- a/test/unit/background-status.test.ts +++ b/test/unit/background-status.test.ts @@ -285,6 +285,9 @@ describe("resumeRun", () => { st.recordRunStart(makeRun({ id: "r1" })); st.recordRunEnd("r1", "failed", undefined, "boom", { failureKind: "cancelled", + failureCode: "cancelled", + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", failedStageId: "s1", resumable: false, }); @@ -296,6 +299,65 @@ describe("resumeRun", () => { assert.match(result.message ?? "", /not resumable/); } }); + + test("killed run returns not_resumable even without explicit resumable metadata", () => { + const st = createStore(); + st.recordRunStart(makeRun({ id: "r1" })); + st.recordRunEnd("r1", "killed", undefined, "bad key", { + failureKind: "auth", + failureCode: "invalid_api_key", + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", + failedStageId: "s1", + resumable: false, + }); + const result = resumeRun("r1", { store: st }); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.mode, "not_resumable"); + assert.equal(result.snapshot.status, "killed"); + assert.match(result.message ?? "", /not resumable/); + } + }); + + test("active blocked recoverable run returns a resumable snapshot message", () => { + const st = createStore(); + st.recordRunStart(makeRun({ id: "r1" })); + st.recordStageStart("r1", { + id: "s1", + name: "limited", + status: "failed", + parentIds: [], + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + toolEvents: [], + }); + st.recordRunBlocked("r1", "rate limit", { + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + failedStageId: "s1", + resumable: true, + retryAfterMs: 1000, + blockedAt: 1234, + }); + + const result = resumeRun("r1", { store: st }); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.mode, "snapshot"); + assert.equal(result.snapshot.status, "running"); + assert.equal(result.snapshot.endedAt, undefined); + assert.equal(result.snapshot.failureCode, "rate_limited"); + assert.equal(result.snapshot.failureRecoverability, "recoverable"); + assert.match(result.message ?? "", /blocked on a recoverable rate_limited failure/); + } + }); }); // --------------------------------------------------------------------------- diff --git a/test/unit/executor.test.ts b/test/unit/executor.test.ts index ac8066835..42c4a8b3e 100644 --- a/test/unit/executor.test.ts +++ b/test/unit/executor.test.ts @@ -12,7 +12,12 @@ import { resolveInputs, } from "../../packages/workflows/src/runs/foreground/executor.js"; import { createStore } from "../../packages/workflows/src/shared/store.js"; -import { WORKFLOW_AUTH_FAILURE_MESSAGE } from "../../packages/workflows/src/shared/workflow-failures.js"; +import { + WORKFLOW_AUTH_FAILURE_MESSAGE, + WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE, + WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE, + WORKFLOW_UNKNOWN_MODEL_MESSAGE, +} from "../../packages/workflows/src/shared/workflow-failures.js"; import { defineWorkflow } from "../../packages/workflows/src/workflows/define-workflow.js"; import { createRegistry } from "../../packages/workflows/src/workflows/registry.js"; import type { AgentSession, CreateAgentSessionOptions } from "@bastani/atomic"; @@ -1442,7 +1447,7 @@ describe("executor.run", () => { prompt: async (text) => { firstRunCalls.push(text); if (text.startsWith("second:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "first-result"; }, }, @@ -1532,7 +1537,7 @@ describe("executor.run", () => { prompt: async (text) => { firstRunCalls.push(text); if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "child-ok"; }, }, @@ -1620,7 +1625,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "child-ok"; }, }, @@ -1703,7 +1708,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "unexpected"; }, }, @@ -1830,7 +1835,7 @@ describe("executor.run", () => { prompt: async (text) => { firstRunCalls.push(text); if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return `child-${firstRunCalls.length}`; }, }, @@ -1930,7 +1935,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "child-first"; }, }, @@ -1982,7 +1987,7 @@ describe("executor.run", () => { assert.deepEqual(after.parentIds, [boundary.id]); }); - test("auth stage failures surface workflow login guidance and preserve details", async () => { + test("missing API key stage failures leave the run active-blocked and resumable", async () => { const st = createStore(); const def = defineWorkflow("auth-fail-wf") .run(async (ctx) => { @@ -2006,23 +2011,515 @@ describe("executor.run", () => { }, ); - assert.equal(wfResult.status, "failed"); - assert.equal( - wfResult.error, - "You must be logged in to run workflows. Run /login and try again.", - ); + assert.equal(wfResult.status, "running"); + assert.equal(wfResult.error, WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE); const storedRun = st.runs()[0]!; const stage = storedRun.stages[0]!; - assert.equal( - stage.error, - "You must be logged in to run workflows. Run /login and try again.", - ); + assert.equal(storedRun.status, "running"); + assert.equal(storedRun.endedAt, undefined); + assert.equal(stage.status, "failed"); + assert.equal(stage.error, WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE); assert.equal(stage.failureKind, "auth"); + assert.equal(stage.failureCode, "missing_api_key"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); assert.equal(stage.failureMessage, "No API key found for provider"); assert.equal(storedRun.failureKind, "auth"); + assert.equal(storedRun.failureCode, "missing_api_key"); + assert.equal(storedRun.failureRecoverability, "recoverable"); + assert.equal(storedRun.failureDisposition, "active_blocked"); assert.equal(storedRun.failureMessage, "No API key found for provider"); assert.equal(storedRun.failedStageId, stage.id); assert.equal(storedRun.resumable, true); + assert.equal(typeof storedRun.blockedAt, "number"); + }); + + test("local login wrapper 401 stage failures leave the run active-blocked and resumable", async () => { + const st = createStore(); + const def = defineWorkflow("local-login-401-wf") + .run(async (ctx) => { + await ctx.stage("needs-login").prompt("x"); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 401, message: "Please log in to continue" }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "running"); + assert.equal(wfResult.error, WORKFLOW_AUTH_FAILURE_MESSAGE); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "running"); + assert.equal(storedRun.endedAt, undefined); + assert.equal(storedRun.resumable, true); + assert.equal(storedRun.failureKind, "auth"); + assert.equal(storedRun.failureCode, "login_required"); + assert.equal(storedRun.failureRecoverability, "recoverable"); + assert.equal(storedRun.failureDisposition, "active_blocked"); + assert.equal(storedRun.failureMessage, "Please log in to continue"); + assert.equal(storedRun.failedStageId, stage.id); + assert.equal(typeof storedRun.blockedAt, "number"); + assert.equal(stage.status, "failed"); + assert.equal(stage.error, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(stage.failureCode, "login_required"); + assert.equal(stage.failureDisposition, "active_blocked"); + }); + + test("invalid provider credential stage failures kill the run and refuse resume", async () => { + const st = createStore(); + const def = defineWorkflow("invalid-key-fail-wf") + .run(async (ctx) => { + await ctx.stage("bad-key").prompt("x"); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { + status: 401, + code: "invalid_api_key", + message: "Incorrect API key provided", + }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "killed"); + assert.equal(wfResult.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "killed"); + assert.notEqual(storedRun.endedAt, undefined); + assert.equal(storedRun.resumable, false); + assert.equal(storedRun.failureKind, "auth"); + assert.equal(storedRun.failureCode, "invalid_api_key"); + assert.equal(storedRun.failureRecoverability, "non_recoverable"); + assert.equal(storedRun.failureDisposition, "terminal_killed"); + assert.equal(storedRun.failedStageId, stage.id); + assert.equal(stage.status, "failed"); + assert.equal(stage.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + assert.equal(stage.failureCode, "invalid_api_key"); + }); + + test("outer invalid credentials after a caught rate-limited stage kill the run", async () => { + const st = createStore(); + const def = defineWorkflow("caught-rate-limit-outer-401-wf") + .run(async (ctx) => { + try { + await ctx.stage("limited").prompt("limited"); + } catch { + // The stage failure is intentionally caught; the outer error + // must still participate in run-level disposition selection. + } + throw { status: 401, message: "Unauthorized" }; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 429, message: "stage rate limited" }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "killed"); + assert.equal(wfResult.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "killed"); + assert.notEqual(storedRun.endedAt, undefined); + assert.equal(storedRun.blockedAt, undefined); + assert.equal(storedRun.resumable, false); + assert.equal(storedRun.failureKind, "auth"); + assert.equal(storedRun.failureCode, "invalid_api_key"); + assert.equal(storedRun.failureRecoverability, "non_recoverable"); + assert.equal(storedRun.failureDisposition, "terminal_killed"); + assert.equal(storedRun.failedStageId, undefined); + assert.equal(stage.status, "failed"); + assert.equal(stage.failureCode, "rate_limited"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); + }); + + test("aggregate invalid credentials after a caught rate-limited stage use aggregate metadata", async () => { + const st = createStore(); + const calls: Array<{ type: string; payload: Record }> = []; + const persistence = { + appendEntry(type: string, payload: Record): string { + calls.push({ type, payload }); + return `entry-${calls.length}`; + }, + setLabel(_entryId: string, _label: string): void {}, + }; + const rawSecret = "sk-testsecret1234567890"; + const def = defineWorkflow("caught-rate-limit-aggregate-invalid-key-wf") + .run(async (ctx) => { + try { + await ctx.stage("limited").prompt("limited"); + } catch { + // Continue to the aggregate provider credential failure. + } + throw new AggregateError([ + { status: 401, message: `Incorrect API key provided: ${rawSecret}` }, + ], "atomic-workflows: 1 parallel step failed"); + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 429, message: "stage rate limited" }; + }, + }, + }, + store: st, + persistence, + }, + ); + + assert.equal(wfResult.status, "killed"); + assert.equal(wfResult.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "killed"); + assert.equal(storedRun.failureKind, "auth"); + assert.equal(storedRun.failureCode, "invalid_api_key"); + assert.equal(storedRun.failureRecoverability, "non_recoverable"); + assert.equal(storedRun.failureDisposition, "terminal_killed"); + assert.equal(storedRun.failedStageId, undefined); + assert.match(storedRun.failureMessage ?? "", /Incorrect API key/); + assert.equal(storedRun.failureMessage?.includes(rawSecret), false); + assert.notEqual(storedRun.failureMessage, stage.failureMessage); + assert.equal(stage.status, "failed"); + assert.equal(stage.failureCode, "rate_limited"); + assert.equal(stage.failureDisposition, "active_blocked"); + + const runEnd = calls.find((call) => call.type === "workflow.run.end")!; + assert.equal(runEnd.payload["failedStageId"], undefined); + assert.equal(runEnd.payload["failureCode"], "invalid_api_key"); + assert.equal(String(runEnd.payload["failureMessage"] ?? "").includes(rawSecret), false); + assert.equal(JSON.stringify({ wfResult, runs: st.runs(), calls }).includes(rawSecret), false); + }); + + test("aggregate ordinary errors after a caught rate-limited stage do not inherit stale rate-limit metadata", async () => { + const st = createStore(); + const def = defineWorkflow("caught-rate-limit-aggregate-error-wf") + .run(async (ctx) => { + try { + await ctx.stage("limited").prompt("limited"); + } catch { + // Continue to an aggregate domain failure. + } + throw new AggregateError([ + new Error("aggregate domain terminal"), + ], "atomic-workflows: 1 parallel step failed"); + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 429, message: "stage rate limited" }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "failed"); + assert.match(wfResult.error ?? "", /atomic-workflows: 1 parallel step failed/); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "failed"); + assert.equal(storedRun.blockedAt, undefined); + assert.equal(storedRun.failureKind, "unknown"); + assert.equal(storedRun.failureCode, "unknown"); + assert.equal(storedRun.failureDisposition, "terminal_failed"); + assert.notEqual(storedRun.failureDisposition, "active_blocked"); + assert.equal(storedRun.failureMessage, "aggregate domain terminal"); + assert.notEqual(storedRun.failureMessage, "stage rate limited"); + assert.equal(storedRun.failedStageId, undefined); + assert.equal(stage.status, "failed"); + assert.equal(stage.failureCode, "rate_limited"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); + }); + + test("outer ordinary errors after a caught rate-limited stage fail with outer error text", async () => { + const st = createStore(); + const def = defineWorkflow("caught-rate-limit-outer-error-wf") + .run(async (ctx) => { + try { + await ctx.stage("limited").prompt("limited"); + } catch { + // Continue to the workflow-level validation failure. + } + throw new Error("outer domain validation failed"); + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 429, message: "stage rate limited" }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "failed"); + assert.equal(wfResult.error, "outer domain validation failed"); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "failed"); + assert.equal(storedRun.error, "outer domain validation failed"); + assert.notEqual(storedRun.endedAt, undefined); + assert.equal(storedRun.blockedAt, undefined); + assert.equal(storedRun.failureKind, "unknown"); + assert.equal(storedRun.failureCode, "unknown"); + assert.equal(storedRun.failureDisposition, "terminal_failed"); + assert.notEqual(storedRun.failureDisposition, "active_blocked"); + assert.equal(storedRun.failureMessage, "outer domain validation failed"); + assert.equal(storedRun.failedStageId, undefined); + assert.equal(stage.status, "failed"); + assert.equal(stage.failureCode, "rate_limited"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); + }); + + test("outer rate limits after a caught rate-limited stage keep the run active-blocked", async () => { + const st = createStore(); + const def = defineWorkflow("caught-rate-limit-outer-429-wf") + .run(async (ctx) => { + try { + await ctx.stage("limited").prompt("limited"); + } catch { + // Both observed failures are recoverable rate limits. + } + throw { status: 429, message: "outer rate limited" }; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { status: 429, message: "stage rate limited" }; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "running"); + const storedRun = st.runs()[0]!; + const stage = storedRun.stages[0]!; + assert.equal(storedRun.status, "running"); + assert.equal(storedRun.endedAt, undefined); + assert.equal(typeof storedRun.blockedAt, "number"); + assert.equal(storedRun.resumable, true); + assert.equal(storedRun.failureKind, "rate_limit"); + assert.equal(storedRun.failureCode, "rate_limited"); + assert.equal(storedRun.failureRecoverability, "recoverable"); + assert.equal(storedRun.failureDisposition, "active_blocked"); + assert.equal(storedRun.failedStageId, stage.id); + assert.equal(stage.status, "failed"); + assert.equal(stage.failureCode, "rate_limited"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); + }); + + test("non-fail-fast parallel invalid provider credentials kill the run", async () => { + const st = createStore(); + const def = defineWorkflow("parallel-invalid-key-wf") + .run(async (ctx) => { + await ctx.parallel( + [ + { name: "ok", prompt: "ok" }, + { name: "bad-key", prompt: "bad-key" }, + ], + { concurrency: 2, failFast: false }, + ); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async (text) => { + if (text === "bad-key") { + throw { status: 401, message: "Unauthorized" }; + } + return "ok"; + }, + }, + }, + store: st, + }, + ); + + assert.equal(wfResult.status, "killed"); + assert.equal(wfResult.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + const storedRun = st.runs()[0]!; + const badKeyStage = storedRun.stages.find((stage) => stage.name === "bad-key")!; + assert.equal(storedRun.status, "killed"); + assert.equal(storedRun.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + assert.equal(storedRun.failureCode, "invalid_api_key"); + assert.equal(storedRun.failureDisposition, "terminal_killed"); + assert.equal(storedRun.failedStageId, badKeyStage.id); + assert.equal(badKeyStage.status, "failed"); + assert.equal(badKeyStage.failureCode, "invalid_api_key"); + assert.equal(badKeyStage.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + }); + + test("non-fail-fast parallel terminal failures beat recoverable blocked failures", async () => { + const st = createStore(); + const def = defineWorkflow("parallel-mixed-provider-failures-wf") + .run(async (ctx) => { + await ctx.parallel( + [ + { name: "limited", prompt: "limited" }, + { name: "bad-key", prompt: "bad-key" }, + ], + { concurrency: 2, failFast: false }, + ); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async (text) => { + if (text === "limited") { + throw { status: 429, message: "too many requests" }; + } + throw { status: 401, message: "Unauthorized" }; + }, + }, + }, + store: st, + }, + ); + + const storedRun = st.runs()[0]!; + const badKeyStage = storedRun.stages.find((stage) => stage.name === "bad-key")!; + const limitedStage = storedRun.stages.find((stage) => stage.name === "limited")!; + assert.equal(wfResult.status, "killed"); + assert.equal(storedRun.failureCode, "invalid_api_key"); + assert.equal(storedRun.failureDisposition, "terminal_killed"); + assert.equal(storedRun.failedStageId, badKeyStage.id); + assert.equal(badKeyStage.failureDisposition, "terminal_killed"); + assert.equal(limitedStage.failureDisposition, "active_blocked"); + }); + + test("non-fail-fast parallel ordinary failures beat recoverable blocked failures", async () => { + const st = createStore(); + const def = defineWorkflow("parallel-mixed-ordinary-failures-wf") + .run(async (ctx) => { + await ctx.parallel( + [ + { name: "limited", prompt: "limited" }, + { name: "domain", prompt: "domain" }, + ], + { concurrency: 2, failFast: false }, + ); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async (text) => { + if (text === "limited") { + throw { status: 429, message: "too many requests" }; + } + throw new Error("domain model validation failed"); + }, + }, + }, + store: st, + }, + ); + + const storedRun = st.runs()[0]!; + const domainStage = storedRun.stages.find((stage) => stage.name === "domain")!; + const limitedStage = storedRun.stages.find((stage) => stage.name === "limited")!; + assert.equal(wfResult.status, "failed"); + assert.match(wfResult.error ?? "", /atomic-workflows: 2 parallel steps failed/); + assert.equal(storedRun.status, "failed"); + assert.match(storedRun.error ?? "", /atomic-workflows: 2 parallel steps failed/); + assert.notEqual(storedRun.endedAt, undefined); + assert.equal(storedRun.blockedAt, undefined); + assert.equal(storedRun.failureKind, "unknown"); + assert.equal(storedRun.failureCode, "unknown"); + assert.equal(storedRun.failureDisposition, "terminal_failed"); + assert.equal(storedRun.failedStageId, domainStage.id); + assert.equal(storedRun.resumable, true); + assert.equal(domainStage.failureDisposition, "terminal_failed"); + assert.equal(domainStage.failureMessage, "domain model validation failed"); + assert.equal(limitedStage.failureCode, "rate_limited"); + assert.equal(limitedStage.failureDisposition, "active_blocked"); }); test("parallel fail-fast marks slow sibling skipped instead of completed", async () => { @@ -2381,7 +2878,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "before-result"; }, }, @@ -2472,7 +2969,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "before-result"; }, }, @@ -2609,7 +3106,7 @@ describe("executor.run", () => { adapters: { prompt: { prompt: async () => { - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); }, }, }, @@ -2696,7 +3193,7 @@ describe("executor.run", () => { adapters: { prompt: { prompt: async () => { - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); }, }, }, @@ -2788,7 +3285,7 @@ describe("executor.run", () => { adapters: { prompt: { prompt: async () => { - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); }, }, }, @@ -2896,7 +3393,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("second:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "first-result"; }, }, @@ -2962,7 +3459,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("after:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return text.toLowerCase(); }, }, @@ -3041,7 +3538,7 @@ describe("executor.run", () => { if (text === "after") throw new Error("unexpected exact prompt"); if (text.includes(",")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return `${text}:done`; }, }, @@ -3118,7 +3615,7 @@ describe("executor.run", () => { prompt: async (text) => { if (text === "alpha" || text === "beta") return `${text}:done`; - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); }, }, }, @@ -3185,7 +3682,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text === "fail") - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return `${text}:done`; }, }, @@ -3247,7 +3744,7 @@ describe("executor.run", () => { prompt: { prompt: async (text) => { if (text.startsWith("second:")) - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); return "first-result"; }, }, @@ -3323,7 +3820,7 @@ describe("executor.run", () => { firstRunCalls.push(text); if (text === "fail-once" && failOnce) { failOnce = false; - throw new Error("rate limit exceeded"); + throw new Error("continuation test failure"); } return `${text}:ok`; }, @@ -3377,7 +3874,7 @@ describe("executor.run", () => { assert.equal(replayed.replayedFromStageId, completed.id); }); - test("failed fallback attempts are recorded on the stage snapshot", async () => { + test("rate-limited fallback attempts are recorded on the active-blocked stage snapshot", async () => { const def = defineWorkflow("failed-fallback-metadata") .output("ok", Type.Boolean()) .run(async (ctx) => { @@ -3419,7 +3916,10 @@ describe("executor.run", () => { }, ); - assert.equal(result.status, "failed"); + assert.equal(result.status, "running"); + assert.equal(result.stages[0]?.status, "failed"); + assert.equal(result.stages[0]?.failureDisposition, "active_blocked"); + assert.equal(result.stages[0]?.failureRecoverability, "recoverable"); assert.deepEqual(result.stages[0]?.attemptedModels, [ "anthropic/primary", "openai/fallback", @@ -3812,10 +4312,12 @@ describe("executor.run", () => { }, ); - assert.equal(result.status, "failed"); - assert.match(result.error ?? "", /missing-model \(not available\)/); + assert.equal(result.status, "killed"); + assert.equal(result.error, WORKFLOW_UNKNOWN_MODEL_MESSAGE); assert.equal(creates, 0); assert.equal(result.stages[0]?.status, "failed"); + assert.equal(result.stages[0]?.failureCode, "unknown_model"); + assert.match(result.stages[0]?.failureMessage ?? "", /missing-model \(not available\)/); }); test("provider-qualified stage model absent from the catalog is trusted and creates a session", async () => { @@ -4209,7 +4711,7 @@ describe("direct SDK helpers", () => { assert.equal(details.warnings, undefined); }); - test("runTask reports classified auth guidance for direct stage failures", async () => { + test("runTask reports classified invalid credential guidance for direct stage failures", async () => { const details = await runTask( { name: "scout", prompt: "inspect repo" }, {}, @@ -4233,8 +4735,8 @@ describe("direct SDK helpers", () => { }, ); - assert.equal(details.status, "failed"); - assert.equal(details.error, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(details.status, "killed"); + assert.equal(details.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); }); test("runTask invalid fallback model fails before session and output side effects", async () => { @@ -4274,7 +4776,7 @@ describe("direct SDK helpers", () => { ); assert.equal(details.status, "failed"); - assert.match(details.error ?? "", /missing-model \(not available\)/); + assert.equal(details.error, WORKFLOW_UNKNOWN_MODEL_MESSAGE); assert.equal(creates, 0); assert.throws(() => readFileSync(output, "utf8")); }); @@ -4980,6 +5482,123 @@ describe("executor.run — lifecycle persistence", () => { assert.equal(runEnd.payload["resumable"], true); }); + test("recoverable rate limit persists run.blocked without run.end", async () => { + const { persistence, calls } = makePersistence(); + const st = createStore(); + + const def = defineWorkflow("blocked-persist-wf") + .run(async (ctx) => { + await ctx.stage("limited").prompt("x"); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async () => { + throw { + status: 429, + code: "rate_limit_exceeded", + message: "rate limit", + retryAfterMs: 1234, + }; + }, + }, + }, + store: st, + persistence, + }, + ); + + assert.equal(wfResult.status, "running"); + assert.deepEqual( + calls.map((c) => c.type), + [ + "workflow.run.start", + "workflow.stage.start", + "workflow.stage.end", + "workflow.run.blocked", + ], + ); + const stageEnd = calls.find((c) => c.type === "workflow.stage.end")!; + assert.equal(stageEnd.payload["status"], "failed"); + assert.equal(stageEnd.payload["failureKind"], "rate_limit"); + assert.equal(stageEnd.payload["failureCode"], "rate_limited"); + assert.equal(stageEnd.payload["failureRecoverability"], "recoverable"); + assert.equal(stageEnd.payload["failureDisposition"], "active_blocked"); + assert.equal(stageEnd.payload["retryAfterMs"], 1234); + + const runBlocked = calls.find((c) => c.type === "workflow.run.blocked")!; + assert.equal(runBlocked.payload["runId"], wfResult.runId); + assert.equal(runBlocked.payload["failureKind"], "rate_limit"); + assert.equal(runBlocked.payload["failureCode"], "rate_limited"); + assert.equal(runBlocked.payload["failureRecoverability"], "recoverable"); + assert.equal(runBlocked.payload["failureDisposition"], "active_blocked"); + assert.equal(runBlocked.payload["resumable"], true); + assert.equal(runBlocked.payload["retryAfterMs"], 1234); + assert.equal(typeof runBlocked.payload["failedStageId"], "string"); + assert.equal(calls.some((c) => c.type === "workflow.run.end"), false); + }); + + test("non-fail-fast parallel rate limits persist run.blocked without ending the run", async () => { + const { persistence, calls } = makePersistence(); + const st = createStore(); + + const def = defineWorkflow("parallel-blocked-persist-wf") + .run(async (ctx) => { + await ctx.parallel( + [ + { name: "limited", prompt: "limited" }, + { name: "ok", prompt: "ok" }, + ], + { concurrency: 2, failFast: false }, + ); + return {}; + }) + .compile(); + + const wfResult = await run( + def, + {}, + { + adapters: { + prompt: { + prompt: async (text) => { + if (text === "limited") { + throw { + status: 429, + code: "rate_limit_exceeded", + message: "rate limit", + retryAfterMs: 2500, + }; + } + return "ok"; + }, + }, + }, + store: st, + persistence, + }, + ); + + const storedRun = st.runs()[0]!; + assert.equal(wfResult.status, "running"); + assert.equal(storedRun.status, "running"); + assert.equal(storedRun.endedAt, undefined); + assert.equal(storedRun.failureCode, "rate_limited"); + assert.equal(storedRun.failureDisposition, "active_blocked"); + assert.equal(calls.some((c) => c.type === "workflow.run.end"), false); + const runBlocked = calls.find((c) => c.type === "workflow.run.blocked")!; + assert.equal(runBlocked.payload["failureKind"], "rate_limit"); + assert.equal(runBlocked.payload["failureCode"], "rate_limited"); + assert.equal(runBlocked.payload["failureDisposition"], "active_blocked"); + assert.equal(runBlocked.payload["retryAfterMs"], 2500); + }); + test("fail-fast skipped queued parallel stages persist start before end", async () => { const { persistence, calls } = makePersistence(); const st = createStore(); diff --git a/test/unit/overlay-graph.test.ts b/test/unit/overlay-graph.test.ts index ba32cc9a6..3708725bd 100644 --- a/test/unit/overlay-graph.test.ts +++ b/test/unit/overlay-graph.test.ts @@ -125,6 +125,7 @@ function makeStore(snap: StoreSnapshot): Store { recordStageInputRequest: () => false, clearStageInputRequest: () => false, recordRunEnd: () => false, + recordRunBlocked: () => false, removeRun: () => false, recordNotice: () => {}, ackNotice: () => false, diff --git a/test/unit/persistence-restore.test.ts b/test/unit/persistence-restore.test.ts index 76fe144c5..985551b02 100644 --- a/test/unit/persistence-restore.test.ts +++ b/test/unit/persistence-restore.test.ts @@ -299,9 +299,13 @@ describe("restoreOnSessionStart", () => { status: "failed", error: "rate limit", failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "terminal_failed", failureMessage: "429 too many requests", failedStageId: "s1", resumable: true, + retryAfterMs: 2000, ts: 3, }, }, @@ -312,9 +316,13 @@ describe("restoreOnSessionStart", () => { assert.equal(run.status, "failed"); assert.equal(run.error, "rate limit"); assert.equal(run.failureKind, "rate_limit"); + assert.equal(run.failureCode, "rate_limited"); + assert.equal(run.failureRecoverability, "recoverable"); + assert.equal(run.failureDisposition, "terminal_failed"); assert.equal(run.failureMessage, "429 too many requests"); assert.equal(run.failedStageId, "s1"); assert.equal(run.resumable, true); + assert.equal(run.retryAfterMs, 2000); }); test("restores completed terminal runs from run.end entries", () => { @@ -374,7 +382,11 @@ describe("restoreOnSessionStart", () => { status: "failed", error: "You must be logged in to run workflows. Run /login and try again.", failureKind: "auth", + failureCode: "missing_api_key", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", failureMessage: "No API key found", + retryAfterMs: 1000, durationMs: 100, }, }, @@ -384,6 +396,131 @@ describe("restoreOnSessionStart", () => { assert.equal(stage.status, "failed"); assert.equal(stage.error, "You must be logged in to run workflows. Run /login and try again."); assert.equal(stage.failureKind, "auth"); + assert.equal(stage.failureCode, "missing_api_key"); + assert.equal(stage.failureRecoverability, "recoverable"); + assert.equal(stage.failureDisposition, "active_blocked"); assert.equal(stage.failureMessage, "No API key found"); + assert.equal(stage.retryAfterMs, 1000); + }); + + test("restores workflow.run.blocked as active recoverable state for any resumeInFlight policy", () => { + const st = createStore(); + const entries: SessionEntry[] = [ + { id: "e1", type: "workflow.run.start", payload: { runId: "r1", name: "wf", inputs: {}, ts: 1 } }, + { id: "e2", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s1", name: "fetch", parentIds: [], ts: 2 } }, + { + id: "e3", + type: "workflow.stage.end", + payload: { + runId: "r1", + stageId: "s1", + status: "failed", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + retryAfterMs: 5000, + }, + }, + { id: "e4", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s2", name: "after", parentIds: ["s1"], ts: 3 } }, + { + id: "e5", + type: "workflow.run.blocked", + payload: { + runId: "r1", + failedStageId: "s1", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureMessage: "HTTP 429", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + retryAfterMs: 5000, + resumable: true, + ts: 4, + }, + }, + ]; + + restoreOnSessionStart(makeSessionManager(entries), { resumeInFlight: "never", persistRuns: true }, st); + + const run = st.runs()[0]!; + assert.equal(run.status, "running"); + assert.equal(run.endedAt, undefined); + assert.equal(run.error, "rate limit"); + assert.equal(run.failureKind, "rate_limit"); + assert.equal(run.failureCode, "rate_limited"); + assert.equal(run.failureRecoverability, "recoverable"); + assert.equal(run.failureDisposition, "active_blocked"); + assert.equal(run.failureMessage, "HTTP 429"); + assert.equal(run.failedStageId, "s1"); + assert.equal(run.resumable, true); + assert.equal(run.retryAfterMs, 5000); + assert.equal(run.blockedAt, 4); + assert.equal(run.stages[0]!.status, "failed"); + assert.equal(run.stages[1]!.status, "blocked"); + assert.equal(run.stages[1]!.blockedByStageId, "s1"); + }); + + test("restores workflow.run.blocked only onto descendants of the failed stage", () => { + const st = createStore(); + const entries: SessionEntry[] = [ + { id: "e1", type: "workflow.run.start", payload: { runId: "r1", name: "wf", inputs: {}, ts: 1 } }, + { id: "e2", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s1", name: "failed", parentIds: [], ts: 2 } }, + { + id: "e3", + type: "workflow.stage.end", + payload: { + runId: "r1", + stageId: "s1", + status: "failed", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + }, + }, + { id: "e4", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s2", name: "unrelated", parentIds: [], ts: 3 } }, + { id: "e5", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s3", name: "direct", parentIds: ["s1"], ts: 4 } }, + { id: "e6", type: "workflow.stage.start", payload: { runId: "r1", stageId: "s4", name: "transitive", parentIds: ["s3"], ts: 5 } }, + { + id: "e7", + type: "workflow.run.blocked", + payload: { + runId: "r1", + failedStageId: "s1", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureMessage: "HTTP 429", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + resumable: true, + ts: 6, + }, + }, + ]; + + restoreOnSessionStart(makeSessionManager(entries), { resumeInFlight: "never", persistRuns: true }, st); + + const run = st.runs()[0]!; + const byId = new Map(run.stages.map((stage) => [stage.id, stage])); + const s1 = byId.get("s1")!; + const s2 = byId.get("s2")!; + const s3 = byId.get("s3")!; + const s4 = byId.get("s4")!; + + assert.equal(run.status, "running"); + assert.equal(s1.status, "failed"); + assert.equal(s2.status, "running"); + assert.equal(s2.blockedByStageId, undefined); + assert.equal(s3.status, "blocked"); + assert.equal(s3.blockedByStageId, "s1"); + assert.equal(s4.status, "blocked"); + assert.equal(s4.blockedByStageId, "s1"); }); }); diff --git a/test/unit/persistence-session-entries.test.ts b/test/unit/persistence-session-entries.test.ts index 4f72c2b9a..ca76db47e 100644 --- a/test/unit/persistence-session-entries.test.ts +++ b/test/unit/persistence-session-entries.test.ts @@ -11,6 +11,7 @@ import { appendStageProgress, appendStageEnd, appendRunEnd, + appendRunBlocked, } from "../../packages/workflows/src/shared/persistence-session-entries.js"; import type { PersistenceAPI } from "../../packages/workflows/src/shared/persistence-session-entries.js"; @@ -244,13 +245,21 @@ describe("appendStageEnd", () => { status: "failed", error: "login required", failureKind: "auth", + failureCode: "missing_api_key", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", failureMessage: "No API key found", + retryAfterMs: 5000, skippedReason: "fail-fast", }); const p = api._entries[0]!.payload; assert.equal(p["error"], "login required"); assert.equal(p["failureKind"], "auth"); + assert.equal(p["failureCode"], "missing_api_key"); + assert.equal(p["failureRecoverability"], "recoverable"); + assert.equal(p["failureDisposition"], "active_blocked"); assert.equal(p["failureMessage"], "No API key found"); + assert.equal(p["retryAfterMs"], 5000); assert.equal(p["skippedReason"], "fail-fast"); }); @@ -352,17 +361,74 @@ describe("appendRunEnd", () => { status: "failed", error: "login required", failureKind: "auth", + failureCode: "invalid_api_key", + failureRecoverability: "non_recoverable", + failureDisposition: "terminal_killed", failureMessage: "No API key found", failedStageId: "s1", - resumable: true, + resumable: false, + retryAfterMs: 7000, ts: 1, }); const p = api._entries[0]!.payload; assert.equal(p["error"], "login required"); assert.equal(p["failureKind"], "auth"); + assert.equal(p["failureCode"], "invalid_api_key"); + assert.equal(p["failureRecoverability"], "non_recoverable"); + assert.equal(p["failureDisposition"], "terminal_killed"); assert.equal(p["failureMessage"], "No API key found"); assert.equal(p["failedStageId"], "s1"); + assert.equal(p["resumable"], false); + assert.equal(p["retryAfterMs"], 7000); + }); + + test("strips active-blocked disposition from terminal failed run.end payloads", () => { + const api = makeMockApi(); + appendRunEnd(api, { + runId: "r1", + status: "failed", + error: "too many requests", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "too many requests", + failedStageId: "s1", + resumable: true, + retryAfterMs: 1000, + ts: 1, + }); + + const p = api._entries[0]!.payload; + assert.equal(p["status"], "failed"); + assert.equal(p["failureKind"], "rate_limit"); + assert.equal(p["failureCode"], "rate_limited"); + assert.equal(p["failureRecoverability"], "recoverable"); + assert.equal("failureDisposition" in p, false); assert.equal(p["resumable"], true); + assert.equal(p["retryAfterMs"], 1000); + }); + + test("normalizes killed run.end payloads to terminal-killed and non-resumable", () => { + const api = makeMockApi(); + appendRunEnd(api, { + runId: "r1", + status: "killed", + error: "workflow killed", + failureKind: "cancelled", + failureCode: "cancelled", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "workflow killed", + resumable: true, + ts: 1, + }); + + const p = api._entries[0]!.payload; + assert.equal(p["status"], "killed"); + assert.equal(p["failureRecoverability"], "non_recoverable"); + assert.equal(p["failureDisposition"], "terminal_killed"); + assert.equal(p["resumable"], false); }); test("no-op when appendEntry absent", () => { @@ -370,3 +436,53 @@ describe("appendRunEnd", () => { appendRunEnd(api, { runId: "r1", status: "completed", ts: 1 }); }); }); + +// --------------------------------------------------------------------------- +// appendRunBlocked +// --------------------------------------------------------------------------- + +describe("appendRunBlocked", () => { + test("calls appendEntry with workflow.run.blocked type and metadata", () => { + const api = makeMockApi(); + appendRunBlocked(api, { + runId: "r1", + failedStageId: "s1", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureMessage: "HTTP 429", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + retryAfterMs: 2500, + resumable: true, + ts: 123, + }); + + assert.equal(api._entries[0]!.type, "workflow.run.blocked"); + const p = api._entries[0]!.payload; + assert.equal(p["runId"], "r1"); + assert.equal(p["failedStageId"], "s1"); + assert.equal(p["error"], "rate limit"); + assert.equal(p["failureKind"], "rate_limit"); + assert.equal(p["failureCode"], "rate_limited"); + assert.equal(p["failureMessage"], "HTTP 429"); + assert.equal(p["failureRecoverability"], "recoverable"); + assert.equal(p["failureDisposition"], "active_blocked"); + assert.equal(p["retryAfterMs"], 2500); + assert.equal(p["resumable"], true); + assert.equal(p["ts"], 123); + }); + + test("no-op when appendEntry absent", () => { + const api: PersistenceAPI = {}; + appendRunBlocked(api, { + runId: "r1", + failedStageId: "s1", + error: "rate limit", + failureKind: "rate_limit", + failureRecoverability: "recoverable", + resumable: true, + ts: 1, + }); + }); +}); diff --git a/test/unit/runtime.test.ts b/test/unit/runtime.test.ts index 4feb6a292..f64e76bed 100644 --- a/test/unit/runtime.test.ts +++ b/test/unit/runtime.test.ts @@ -22,6 +22,7 @@ import { defineWorkflow } from "../../packages/workflows/src/workflows/define-wo import { Type } from "typebox"; import { createStore } from "../../packages/workflows/src/shared/store.js"; import { renderResult } from "../../packages/workflows/src/extension/render-result.js"; +import { WORKFLOW_UNKNOWN_MODEL_MESSAGE } from "../../packages/workflows/src/shared/workflow-failures.js"; import { NON_INTERACTIVE_WORKFLOW_POLICY } from "../../packages/workflows/src/shared/types.js"; import type { WorkflowDefinition, @@ -274,7 +275,7 @@ describe("runtime.runDirect — workflow intercom", () => { }); assert.equal(result.status, "failed"); - assert.match(result.error ?? "", /missing-model \(not available\)/); + assert.equal(result.error, WORKFLOW_UNKNOWN_MODEL_MESSAGE); assert.equal(activeStore.runs().length, 0); }); diff --git a/test/unit/slash-dispatch.test.ts b/test/unit/slash-dispatch.test.ts index 5aebed3e0..c37129ffb 100644 --- a/test/unit/slash-dispatch.test.ts +++ b/test/unit/slash-dispatch.test.ts @@ -38,15 +38,22 @@ import type { import { createRegistry } from "../../packages/workflows/src/workflows/registry.js"; import { defineWorkflow } from "../../packages/workflows/src/workflows/define-workflow.js"; import { Type } from "typebox"; -import type { WorkflowDefinition } from "../../packages/workflows/src/shared/types.js"; +import type { + WorkflowDefinition, + WorkflowPersistencePort, +} from "../../packages/workflows/src/shared/types.js"; import { createExtensionRuntime, type ExtensionRuntime, } from "../../packages/workflows/src/extension/runtime.js"; import type { ChatSurfacePayload } from "../../packages/workflows/src/tui/chat-surface-message.js"; import { store } from "../../packages/workflows/src/shared/store.js"; +import { + restoreOnSessionStart, + type SessionEntry, +} from "../../packages/workflows/src/shared/persistence-restore.js"; import { WORKFLOW_STAGE_SUBAGENT_GUARD_ENV } from "@bastani/atomic"; -import { WORKFLOW_AUTH_FAILURE_MESSAGE } from "../../packages/workflows/src/shared/workflow-failures.js"; +import { WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE } from "../../packages/workflows/src/shared/workflow-failures.js"; import { LIFECYCLE_NOTICE_CUSTOM_TYPE } from "../../packages/workflows/src/extension/lifecycle-notifications.js"; import type { PiCustomComponent, @@ -3583,7 +3590,7 @@ export default defineWorkflow("tool-headless-lifecycle") }); assert.equal(result.status, "failed"); - assert.equal(result.error, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(result.error, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); }); test("makeExecuteWorkflowTool resume rejects ambiguous stage prefixes", async () => { @@ -3737,6 +3744,293 @@ export default defineWorkflow("tool-headless-lifecycle") ); }); + test("makeExecuteWorkflowTool resume starts linked continuation for active blocked recoverable workflow", async () => { + const sourceRunId = `resume-tool-blocked-${Date.now()}`; + const def = defineWorkflow("tool-resume-blocked-wf") + .output("first", Type.Optional(Type.Any())) + .output("second", Type.Optional(Type.Any())) + .run(async (ctx) => { + const first = await ctx.stage("first").prompt("first"); + const second = await ctx + .stage("second") + .prompt(`second:${first}`); + return { first, second }; + }) + .compile(); + + store.recordRunStart({ + id: sourceRunId, + name: def.name, + inputs: {}, + status: "running", + startedAt: Date.now(), + stages: [], + }); + store.recordStageStart(sourceRunId, { + id: "blocked-first", + name: "first", + status: "completed", + parentIds: [], + toolEvents: [], + result: "first-old", + }); + store.recordStageEnd(sourceRunId, { + id: "blocked-first", + name: "first", + status: "completed", + parentIds: [], + toolEvents: [], + result: "first-old", + }); + store.recordStageStart(sourceRunId, { + id: "blocked-second", + name: "second", + status: "failed", + parentIds: ["blocked-first"], + toolEvents: [], + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + }); + store.recordStageEnd(sourceRunId, { + id: "blocked-second", + name: "second", + status: "failed", + parentIds: ["blocked-first"], + toolEvents: [], + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + }); + store.recordRunBlocked(sourceRunId, "rate limit", { + resumable: true, + failedStageId: "blocked-second", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + }); + + const calls: string[] = []; + const persistenceCalls: Array<{ + readonly type: string; + readonly payload: Record; + }> = []; + const persistence: WorkflowPersistencePort = { + appendEntry(type, payload) { + persistenceCalls.push({ type, payload }); + return `entry-${persistenceCalls.length}`; + }, + }; + const runtime = createExtensionRuntime({ + registry: createRegistry([def]), + store, + persistence, + adapters: { + prompt: { + prompt: async (text) => { + calls.push(text); + return "second-new"; + }, + }, + }, + }); + const handler = makeExecuteWorkflowTool( + runtime, + () => undefined, + () => undefined, + ); + + const result = await handler( + { action: "resume", runId: sourceRunId }, + {} as never, + ); + + assert.equal(result.action, "resume"); + const r = result as { + action: string; + status: string; + runId: string; + message: string; + }; + assert.equal(r.status, "running"); + assert.notEqual(r.runId, sourceRunId); + assert.match(r.message, /Resuming blocked workflow/); + await jobTracker.get(r.runId)?.promise; + assert.deepEqual(calls, ["second:first-old"]); + const continued = store.runs().find((run) => run.id === r.runId)!; + assert.equal(continued.status, "completed"); + assert.equal(continued.resumedFromRunId, sourceRunId); + assert.equal(continued.stages[0]!.replayed, true); + const source = store.runs().find((run) => run.id === sourceRunId)!; + assert.equal(source.status, "killed"); + assert.equal(source.endedAt !== undefined, true); + assert.equal(source.blockedAt, undefined); + assert.equal(source.resumable, false); + assert.equal(source.failureKind, "rate_limit"); + assert.equal(source.failureCode, "rate_limited"); + assert.equal(source.failureRecoverability, "non_recoverable"); + assert.equal(source.failureDisposition, "terminal_killed"); + assert.equal(source.failureMessage, "HTTP 429"); + assert.equal(source.failedStageId, "blocked-second"); + + const sourceRunEnd = persistenceCalls.find( + (call) => + call.type === "workflow.run.end" && + call.payload["runId"] === sourceRunId, + ); + assert.ok(sourceRunEnd); + assert.equal(sourceRunEnd.payload["status"], "killed"); + assert.equal(sourceRunEnd.payload["resumable"], false); + assert.equal(sourceRunEnd.payload["failureRecoverability"], "non_recoverable"); + assert.equal(sourceRunEnd.payload["failureDisposition"], "terminal_killed"); + }); + + test("makeExecuteWorkflowTool resume finalizes restored blocked source run", async () => { + const sourceRunId = `resume-tool-restored-blocked-${Date.now()}`; + const def = defineWorkflow("tool-resume-restored-blocked-wf") + .output("first", Type.Optional(Type.Any())) + .output("second", Type.Optional(Type.Any())) + .run(async (ctx) => { + const first = await ctx.stage("first").prompt("first"); + const second = await ctx + .stage("second") + .prompt(`second:${first}`); + return { first, second }; + }) + .compile(); + const entries: SessionEntry[] = [ + { + id: "e1", + type: "workflow.run.start", + payload: { runId: sourceRunId, name: def.name, inputs: {}, ts: 1 }, + }, + { + id: "e2", + type: "workflow.stage.start", + payload: { runId: sourceRunId, stageId: "restored-first", name: "first", parentIds: [], ts: 2 }, + }, + { + id: "e3", + type: "workflow.stage.end", + payload: { + runId: sourceRunId, + stageId: "restored-first", + status: "completed", + summary: "first-old", + }, + }, + { + id: "e4", + type: "workflow.stage.start", + payload: { + runId: sourceRunId, + stageId: "restored-second", + name: "second", + parentIds: ["restored-first"], + ts: 3, + }, + }, + { + id: "e5", + type: "workflow.stage.end", + payload: { + runId: sourceRunId, + stageId: "restored-second", + status: "failed", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + failureMessage: "HTTP 429", + }, + }, + { + id: "e6", + type: "workflow.run.blocked", + payload: { + runId: sourceRunId, + failedStageId: "restored-second", + error: "rate limit", + failureKind: "rate_limit", + failureCode: "rate_limited", + failureMessage: "HTTP 429", + failureRecoverability: "recoverable", + failureDisposition: "active_blocked", + resumable: true, + ts: 4, + }, + }, + ]; + restoreOnSessionStart( + { getEntries: () => entries }, + { resumeInFlight: "never", persistRuns: true }, + store, + ); + + const calls: string[] = []; + let markPromptStarted = (): void => {}; + const promptStarted = new Promise((resolve) => { + markPromptStarted = resolve; + }); + let releasePrompt = (_value: string): void => {}; + const promptRelease = new Promise((resolve) => { + releasePrompt = resolve; + }); + const runtime = createExtensionRuntime({ + registry: createRegistry([def]), + store, + adapters: { + prompt: { + prompt: async (text) => { + calls.push(text); + markPromptStarted(); + return promptRelease; + }, + }, + }, + }); + const handler = makeExecuteWorkflowTool( + runtime, + () => undefined, + () => undefined, + ); + + const result = await handler( + { action: "resume", runId: sourceRunId }, + {} as never, + ); + const r = result as { action: string; status: string; runId: string }; + assert.equal(r.action, "resume"); + assert.equal(r.status, "running"); + assert.notEqual(r.runId, sourceRunId); + + await promptStarted; + assert.deepEqual(calls, ["second:first-old"]); + const source = store.runs().find((run) => run.id === sourceRunId)!; + assert.equal(source.status, "killed"); + assert.equal(source.failureDisposition, "terminal_killed"); + assert.equal(source.failureRecoverability, "non_recoverable"); + assert.equal(source.resumable, false); + assert.equal(source.blockedAt, undefined); + const inFlight = store.runs().filter((run) => run.endedAt === undefined); + assert.deepEqual(inFlight.map((run) => run.id), [r.runId]); + + releasePrompt("second-new"); + await jobTracker.get(r.runId)?.promise; + const continued = store.runs().find((run) => run.id === r.runId)!; + assert.equal(continued.status, "completed"); + assert.equal(continued.resumedFromRunId, sourceRunId); + assert.equal(continued.resumeFromStageId, "restored-second"); + }); + test("makeExecuteWorkflowTool resume surfaces workflow_not_found for failed resumable run without registry definition", async () => { const runId = `resume-tool-failed-${Date.now()}`; store.recordRunStart(makeInflightRun(runId)); diff --git a/test/unit/workflow-failures.test.ts b/test/unit/workflow-failures.test.ts index f82264205..f18e39e21 100644 --- a/test/unit/workflow-failures.test.ts +++ b/test/unit/workflow-failures.test.ts @@ -6,66 +6,314 @@ import { describe, test } from "bun:test"; import assert from "node:assert/strict"; import { WORKFLOW_AUTH_FAILURE_MESSAGE, + WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE, + WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE, + WORKFLOW_UNKNOWN_MODEL_MESSAGE, classifyWorkflowFailure, } from "../../packages/workflows/src/shared/workflow-failures.js"; describe("classifyWorkflowFailure", () => { - test("normalizes auth/no-key failures to workflow login guidance", () => { + test("normalizes missing provider key failures to recoverable active-blocked auth", () => { const failure = classifyWorkflowFailure(new Error("No API key found for provider")); assert.equal(failure.kind, "auth"); - assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(failure.code, "missing_api_key"); + assert.equal(failure.userMessage, WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE); assert.equal(failure.message, "No API key found for provider"); assert.equal(failure.retryable, true); assert.equal(failure.resumable, true); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); }); - test("classifies 429/quota failures as resumable rate limits", () => { + test("classifies 429/quota failures as recoverable active-blocked rate limits", () => { const failure = classifyWorkflowFailure(new Error("HTTP 429 quota exceeded")); assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); assert.equal(failure.userMessage, "HTTP 429 quota exceeded"); assert.equal(failure.retryable, true); assert.equal(failure.resumable, true); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + }); + + test("classifies quota-only fallback text as recoverable active-blocked", () => { + const failure = classifyWorkflowFailure(new Error("quota exceeded")); + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "quota_limited"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + }); + + test("classifies string-only rate limit fallback text as recoverable active-blocked", () => { + const failure = classifyWorkflowFailure(new Error("rate limit exceeded")); + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.retryable, true); + assert.equal(failure.resumable, true); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); }); - test("classifies abort errors as non-resumable cancellation", () => { + test("classifies assistant errorMessage rate limit fallback as recoverable active-blocked", () => { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "rate limit exceeded", + }); + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.retryable, true); + assert.equal(failure.resumable, true); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + }); + + test("classifies abort errors as non-resumable terminal cancellation", () => { const failure = classifyWorkflowFailure(new DOMException("workflow killed", "AbortError")); assert.equal(failure.kind, "cancelled"); + assert.equal(failure.code, "cancelled"); assert.equal(failure.retryable, false); assert.equal(failure.resumable, false); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); }); test("classifies provider/model outages separately from auth", () => { const failure = classifyWorkflowFailure(new Error("model provider service unavailable")); assert.equal(failure.kind, "provider"); + assert.equal(failure.code, "provider_unavailable"); assert.equal(failure.retryable, true); assert.equal(failure.resumable, true); + assert.equal(failure.disposition, "active_blocked"); }); test("uses structured HTTP statuses before message fallback", () => { const auth = classifyWorkflowFailure({ message: "request failed", status: 401 }); assert.equal(auth.kind, "auth"); - assert.equal(auth.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(auth.code, "invalid_api_key"); + assert.equal(auth.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); const rateLimit = classifyWorkflowFailure({ message: "request failed", statusCode: 429 }); assert.equal(rateLimit.kind, "rate_limit"); + assert.equal(rateLimit.code, "rate_limited"); assert.equal(rateLimit.retryable, true); const provider = classifyWorkflowFailure({ message: "request failed", status: 503 }); assert.equal(provider.kind, "provider"); + assert.equal(provider.code, "provider_unavailable"); assert.equal(provider.retryable, true); }); + test("lets structured local login codes beat wrapper 401 defaults", () => { + for (const code of ["login_required", "auth_required", "authentication_required", "not_logged_in"] as const) { + const failure = classifyWorkflowFailure({ status: 401, code, message: "wrapper 401" }); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + } + }); + + test("uses auth-required diagnostics before generic wrapper 401 defaults", () => { + const failure = classifyWorkflowFailure({ + status: 401, + message: "provider request failed", + diagnostics: [{ error: { code: "auth_required", message: "Please log in to continue" } }], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.message, "Please log in to continue"); + assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + }); + + test("uses clear local login wrapper-401 messages before provider credential defaults", () => { + for (const message of [ + "Please log in to continue", + "not logged in", + "login required", + "Run /login to continue", + "Authentication failed for \"openai\". Credentials may have expired or network is unavailable. Run '/login openai' to re-authenticate.", + ] as const) { + const failure = classifyWorkflowFailure({ status: 401, message }); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + } + }); + + test("keeps provider 401 auth text classified as invalid provider credentials", () => { + for (const message of ["Unauthorized", "authentication required"]) { + const failure = classifyWorkflowFailure({ status: 401, message }); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("classifies string-only provider auth fallback as invalid credentials", () => { + for (const error of [ + new Error("OpenAI API error (401): Unauthorized"), + new Error("Unauthorized"), + "authentication required", + ] as const) { + const failure = classifyWorkflowFailure(error); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.retryable, false); + assert.equal(failure.resumable, false); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("classifies non-contiguous invalid API key fallback messages as invalid credentials", () => { + for (const message of [ + "The API key provided is invalid", + "The API key you supplied is incorrect", + ] as const) { + const failure = classifyWorkflowFailure(new Error(message)); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.retryable, false); + assert.equal(failure.resumable, false); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("keeps clear string-only local login fallback recoverable", () => { + for (const error of [ + new Error("Run /login to continue"), + new Error("not logged in"), + "login required", + "please login", + "please log in", + "log in to continue", + ] as const) { + const failure = classifyWorkflowFailure(error); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); + assert.equal(failure.retryable, true); + assert.equal(failure.resumable, true); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + } + }); + + test("provider credential messages and causes override broad auth wrapper codes", () => { + const failures = [ + classifyWorkflowFailure({ + status: 401, + code: "auth_required", + message: "Incorrect API key provided", + }), + classifyWorkflowFailure({ + status: 401, + code: "auth_required", + message: "wrapper 401", + cause: { code: "invalid_api_key", message: "Incorrect API key provided" }, + }), + ]; + + for (const failure of failures) { + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("uses missing API key diagnostics before generic wrapper 401 defaults", () => { + const failure = classifyWorkflowFailure({ + status: 401, + message: "provider request failed", + diagnostics: [{ error: { code: "missing_api_key", message: "No API key found" } }], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "missing_api_key"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.message, "No API key found"); + assert.equal(failure.userMessage, WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE); + }); + + test("uses missing API key diagnostics before generic wrapper code 401 defaults", () => { + for (const code of [401, "401"] as const) { + const failure = classifyWorkflowFailure({ + code, + message: "provider request failed", + diagnostics: [{ error: { code: "missing_api_key", message: "No API key found" } }], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "missing_api_key"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.message, "No API key found"); + assert.equal(failure.userMessage, WORKFLOW_MISSING_API_KEY_FAILURE_MESSAGE); + } + }); + + test("keeps generic wrapper code 401 without stronger diagnostics as invalid provider credentials", () => { + for (const code of [401, "401"] as const) { + const failure = classifyWorkflowFailure({ + code, + message: "provider request failed", + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.resumable, false); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + test("uses structured codes and causes before message fallback", () => { const auth = classifyWorkflowFailure({ message: "provider error", code: "AUTH_REQUIRED" }); assert.equal(auth.kind, "auth"); + assert.equal(auth.code, "login_required"); const rateLimit = classifyWorkflowFailure(new Error("outer failure", { cause: { message: "inner failure", code: "rate_limit_exceeded" }, })); assert.equal(rateLimit.kind, "rate_limit"); + assert.equal(rateLimit.code, "rate_limited"); const cancelled = classifyWorkflowFailure({ message: "stopped", code: "AbortError" }); assert.equal(cancelled.kind, "cancelled"); + assert.equal(cancelled.disposition, "terminal_killed"); + }); + + test("treats broad auth wrapper codes as weak when the message names provider credentials", () => { + const failure = classifyWorkflowFailure({ + code: "auth", + message: "Incorrect API key provided", + }); + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); }); test("uses SDK assistant error shapes", () => { @@ -76,7 +324,8 @@ describe("classifyWorkflowFailure", () => { diagnostics: [{ error: { code: 429, message: "quota exceeded" } }], }); assert.equal(failure.kind, "rate_limit"); - assert.equal(failure.message, "provider request failed"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.message, "quota exceeded"); const cancelled = classifyWorkflowFailure({ role: "assistant", @@ -84,6 +333,320 @@ describe("classifyWorkflowFailure", () => { errorMessage: "stream aborted", }); assert.equal(cancelled.kind, "cancelled"); + assert.equal(cancelled.disposition, "terminal_killed"); + }); + + test("classifies OpenAI-style invalid API key diagnostics as terminal killed", () => { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics: [{ + error: { + status: 401, + code: "invalid_api_key", + message: "Incorrect API key provided: sk-testsecret123456789", + }, + }], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.retryable, false); + assert.equal(failure.resumable, false); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + assert.doesNotMatch(failure.message, /sk-testsecret123456789/); + assert.doesNotMatch(failure.userMessage, /sk-testsecret/); + }); + + test("uses diagnostic-only invalid key messages as the decisive failure message", () => { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics: [{ + error: { + message: "Incorrect API key provided", + }, + }], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.message, "Incorrect API key provided"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + }); + + test("classifies diagnostic-only provider 401 unauthorized messages as terminal invalid credentials", () => { + for (const diagnostic of [ + { error: { message: "401 Unauthorized" } }, + { message: "401 Unauthorized" }, + { error: { message: "OpenAI API error (401): Unauthorized" } }, + ] as const) { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics: [diagnostic], + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.retryable, false); + assert.equal(failure.resumable, false); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("lets invalid credential diagnostics beat rate limits regardless of diagnostic order", () => { + const diagnosticSets = [ + [ + { error: { status: 429, message: "too many requests" } }, + { error: { status: 401, code: "invalid_api_key", message: "Incorrect API key provided" } }, + ], + [ + { error: { status: 401, code: "invalid_api_key", message: "Incorrect API key provided" } }, + { error: { status: 429, message: "too many requests" } }, + ], + ] as const; + + for (const diagnostics of diagnosticSets) { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics, + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.resumable, false); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + } + }); + + test("keeps all-recoverable diagnostics active-blocked", () => { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics: [ + { error: { status: 429, message: "too many requests", retryAfterMs: 2500 } }, + { error: { status: 503, message: "provider unavailable" } }, + ], + }); + + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.retryAfterMs, 2500); + }); + + test("preserves retry hints from later all-recoverable diagnostics", () => { + const failure = classifyWorkflowFailure({ + role: "assistant", + stopReason: "error", + errorMessage: "provider request failed", + diagnostics: [ + { error: { status: 503, message: "provider unavailable" } }, + { error: { status: 429, message: "too many requests", retryAfterMs: 2500 } }, + ], + }); + + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.resumable, true); + assert.equal(failure.retryAfterMs, 2500); + }); + + test("classifies AggregateError inner provider failures before wrapper text", () => { + const rateLimited = classifyWorkflowFailure(new AggregateError([ + { status: 429, message: "too many requests" }, + ], "atomic-workflows: 1 parallel step failed")); + assert.equal(rateLimited.kind, "rate_limit"); + assert.equal(rateLimited.code, "rate_limited"); + assert.equal(rateLimited.disposition, "active_blocked"); + + const invalidKey = classifyWorkflowFailure(new AggregateError([ + { status: 401, message: "Unauthorized" }, + ], "atomic-workflows: 1 parallel step failed")); + assert.equal(invalidKey.kind, "auth"); + assert.equal(invalidKey.code, "invalid_api_key"); + assert.equal(invalidKey.disposition, "terminal_killed"); + }); + + test("classifies mixed ordinary and rate-limit aggregate failures as terminal failed", () => { + const failure = classifyWorkflowFailure(new Error("wrapper", { + cause: new AggregateError([ + new Error("domain validation failed"), + { status: 429, message: "too many requests" }, + ], "atomic-workflows: 2 parallel steps failed"), + })); + + assert.equal(failure.kind, "unknown"); + assert.equal(failure.code, "unknown"); + assert.equal(failure.recoverability, "unknown"); + assert.equal(failure.disposition, "terminal_failed"); + assert.equal(failure.resumable, true); + }); + + test("preserves all-recoverable aggregate failures as active-blocked", () => { + const failure = classifyWorkflowFailure(new AggregateError([ + { status: 429, message: "too many requests", retryAfterMs: 2500 }, + { status: 503, message: "provider unavailable" }, + ], "atomic-workflows: 2 parallel steps failed")); + + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.retryAfterMs, 2500); + }); + + test("preserves retry hints from later all-recoverable aggregate failures", () => { + const failure = classifyWorkflowFailure(new AggregateError([ + { status: 503, message: "provider unavailable" }, + { status: 429, message: "too many requests", retryAfterMs: 2500 }, + ], "atomic-workflows: 2 parallel steps failed")); + + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.recoverability, "recoverable"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.retryAfterMs, 2500); + }); + + test("lets invalid credentials win over rate limits in aggregate failures", () => { + const failure = classifyWorkflowFailure(new AggregateError([ + { status: 429, message: "too many requests" }, + { status: 401, message: "Unauthorized" }, + ], "atomic-workflows: 2 parallel steps failed")); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.recoverability, "non_recoverable"); + assert.equal(failure.disposition, "terminal_killed"); + assert.equal(failure.resumable, false); + }); + + test("extracts retry-after metadata from structured rate limits", () => { + const failure = classifyWorkflowFailure({ + message: "slow down", + status: 429, + headers: { "retry-after": "3" }, + }); + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.disposition, "active_blocked"); + assert.equal(failure.retryAfterMs, 3000); + }); + + test("treats bare retryAfter as seconds while explicit retryAfterMs remains milliseconds", () => { + const explicitMs = classifyWorkflowFailure({ + message: "slow down", + status: 429, + retryAfterMs: 2500, + }); + assert.equal(explicitMs.kind, "rate_limit"); + assert.equal(explicitMs.retryAfterMs, 2500); + + const direct = classifyWorkflowFailure({ + message: "slow down", + status: 429, + retryAfter: 3, + }); + assert.equal(direct.kind, "rate_limit"); + assert.equal(direct.retryAfterMs, 3000); + + const seconds = classifyWorkflowFailure({ + message: "slow down", + status: 429, + retryAfterSeconds: 3, + }); + assert.equal(seconds.kind, "rate_limit"); + assert.equal(seconds.retryAfterMs, 3000); + + const header = classifyWorkflowFailure({ + message: "slow down", + status: 429, + "retry-after": "3", + }); + assert.equal(header.kind, "rate_limit"); + assert.equal(header.retryAfterMs, 3000); + }); + + test("structured 429 wins over misleading auth text", () => { + const failure = classifyWorkflowFailure({ + message: "Incorrect API key mentioned in provider retry body", + status: 429, + }); + assert.equal(failure.kind, "rate_limit"); + assert.equal(failure.code, "rate_limited"); + assert.equal(failure.disposition, "active_blocked"); + }); + + test("redacts top-level structured invalid provider credential messages", () => { + for (const secret of [ + "sk-testsecret1234567890", + "api_key=super-secret-value", + "token=super-secret-value", + "credential=super-secret-value", + "secret=super-secret-value", + "Authorization: Bearer secret-token-value", + "Bearer secret-token-value", + ] as const) { + const failure = classifyWorkflowFailure({ + status: 401, + code: "invalid_api_key", + message: `Incorrect API key provided: ${secret}`, + }); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + assert.equal(failure.message.includes(secret), false); + assert.equal(failure.userMessage.includes(secret), false); + assert.match(failure.message, /\[redacted\]/); + } + }); + + test("redacts string-only invalid provider key fallback messages", () => { + const secret = "sk-testsecret1234567890"; + const failure = classifyWorkflowFailure(new Error(`Incorrect API key provided: ${secret}`)); + + assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "invalid_api_key"); + assert.equal(failure.userMessage, WORKFLOW_INVALID_PROVIDER_CREDENTIALS_MESSAGE); + assert.equal(failure.message.includes(secret), false); + assert.match(failure.message, /\[redacted\]/); + }); + + test("redacts sensitive fallback unknown messages", () => { + for (const secret of [ + "api_key=super-secret-value", + "token=super-secret-value", + "credential=super-secret-value", + "secret=super-secret-value", + "Authorization: Bearer secret-token-value", + "Bearer secret-token-value", + ] as const) { + const failure = classifyWorkflowFailure(new Error(`tool failed with ${secret}`)); + assert.equal(failure.kind, "unknown"); + assert.equal(failure.code, "unknown"); + assert.equal(failure.message.includes(secret), false); + assert.equal(failure.userMessage.includes(secret), false); + assert.match(failure.message, /\[redacted\]/); + assert.match(failure.userMessage, /\[redacted\]/); + } }); test("does not treat log information/input errors as auth failures", () => { @@ -95,13 +658,16 @@ describe("classifyWorkflowFailure", () => { assert.equal(failure.kind, "unknown"); assert.equal(failure.userMessage, message); assert.equal(failure.retryable, false); + assert.equal(failure.disposition, "terminal_failed"); } }); test("still treats bounded log in guidance as auth failure", () => { const failure = classifyWorkflowFailure(new Error("Please log in to continue")); assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); + assert.equal(failure.disposition, "active_blocked"); }); test("does not treat generic domain/tool model errors as provider outages", () => { @@ -115,12 +681,19 @@ describe("classifyWorkflowFailure", () => { } }); - test("still treats unavailable or missing model errors as provider outages", () => { - for (const message of ["model unavailable", "model not found"]) { - const failure = classifyWorkflowFailure(new Error(message)); - assert.equal(failure.kind, "provider"); - assert.equal(failure.retryable, true); - } + test("distinguishes unavailable providers from unknown models", () => { + const unavailable = classifyWorkflowFailure(new Error("model unavailable")); + assert.equal(unavailable.kind, "provider"); + assert.equal(unavailable.code, "provider_unavailable"); + assert.equal(unavailable.retryable, true); + + const missing = classifyWorkflowFailure(new Error("model not found")); + assert.equal(missing.kind, "provider"); + assert.equal(missing.code, "unknown_model"); + assert.equal(missing.userMessage, WORKFLOW_UNKNOWN_MODEL_MESSAGE); + assert.equal(missing.retryable, false); + assert.equal(missing.resumable, false); + assert.equal(missing.disposition, "terminal_killed"); }); test("does not treat generic OAuth metadata errors as auth failures", () => { @@ -132,6 +705,7 @@ describe("classifyWorkflowFailure", () => { test("still treats OAuth token errors as auth failures", () => { const failure = classifyWorkflowFailure(new Error("OAuth token expired")); assert.equal(failure.kind, "auth"); + assert.equal(failure.code, "login_required"); assert.equal(failure.userMessage, WORKFLOW_AUTH_FAILURE_MESSAGE); }); });