diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b2af1bfd2f..4ed3f1b6c7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -260,12 +260,15 @@ export type { GenericToolLedgerAppendValidation, ToolLedgerLane, ToolLedgerLaneValidation, + ToolLedgerRejectionCode, ToolLedgerScanOperation, ToolLedgerScanResult, ToolLedgerTransitionKind, ToolLedgerTransitionValidation, } from './tool-ledger-scanner.js'; export { + ToolLedgerCorruptionError, + ToolLedgerRejectionError, scanToolLedger, validateGenericToolLedgerAppend, validateToolLedgerEventLane, diff --git a/packages/core/src/tool-ledger-scanner.ts b/packages/core/src/tool-ledger-scanner.ts index c459917641..b94e05042a 100644 --- a/packages/core/src/tool-ledger-scanner.ts +++ b/packages/core/src/tool-ledger-scanner.ts @@ -31,6 +31,49 @@ export interface ToolLedgerIssue { toolCallId?: string; } +/** + * The ledger refused a CANDIDATE event because that event was wrong — the store + * itself is healthy and still readable. The distinction decides what a run may + * do next: when the ledger can no longer be trusted the run latches its store + * unavailable and fails closed, but latching it for a bad candidate costs the + * run its own terminal write, which is how one refused append left a run stuck + * at `running` with no terminal event at all (#2234). This error is always a + * producer bug: something emitted a fact the ledger's invariants forbid. + * + * It deliberately does NOT cover a ledger that is already corrupt. That refusal + * rejects well-formed candidates because of damage elsewhere in the workspace, + * so "the store is healthy" is false and the run must keep failing closed — + * see `ToolLedgerCorruptionError`, which is a plain durability failure and is + * classified as one. + */ +export class ToolLedgerRejectionError extends Error { + readonly name = 'ToolLedgerRejectionError'; + + constructor( + readonly code: ToolLedgerRejectionCode, + readonly eventId: string, + ) { + super(`Tool ledger transition rejected: ${code} at ${eventId}`); + } +} + +/** + * The ledger the store already holds is corrupt, so it refuses writes that have + * nothing wrong with them. Unlike `ToolLedgerRejectionError` this is not a + * producer bug and the store is not usable: nothing the run emits next can be + * trusted to land, so it stays on the fail-closed path. + */ +export class ToolLedgerCorruptionError extends Error { + readonly name = 'ToolLedgerCorruptionError'; + + constructor( + readonly code: ToolLedgerRejectionCode, + readonly eventId: string, + ) { + super(`Tool ledger is corrupt: ${code} at ${eventId}`); + } +} + export interface ToolLedgerScanOperation { toolCallId: string; toolName?: string; @@ -67,16 +110,19 @@ export type ToolLedgerTransitionKind = | 't2_outcome' | 'recovery_bundle'; +/** Every reason the ledger has for refusing a candidate event. */ +export type ToolLedgerRejectionCode = + | ToolLedgerIssueCode + | 'semantic_lane_conflict' + | 'reserved_tool_boundary_fact' + | 'reserved_recovery_fact' + | 'transition_shape_conflict'; + export type ToolLedgerTransitionValidation = | { ok: true } | { ok: false; - code: - | ToolLedgerIssueCode - | 'semantic_lane_conflict' - | 'reserved_tool_boundary_fact' - | 'reserved_recovery_fact' - | 'transition_shape_conflict'; + code: ToolLedgerRejectionCode; eventId: string; operationId?: string; toolCallId?: string; diff --git a/packages/runtime/src/__tests__/agent-swarm-tools.test.ts b/packages/runtime/src/__tests__/agent-swarm-tools.test.ts index d87c438336..c3d14db6d4 100644 --- a/packages/runtime/src/__tests__/agent-swarm-tools.test.ts +++ b/packages/runtime/src/__tests__/agent-swarm-tools.test.ts @@ -160,10 +160,44 @@ describe('AgentSwarm adapter', () => { ); assert.equal(starts, 0); - assert.match( - String((result as { error?: unknown }).error), - /Provide exactly one of subagent_id or legacy profile/, + const refusal = String((result as { error?: unknown }).error); + assert.match(refusal, /Neither subagent_id nor profile is set, so no child is selected/); + // The refusal has to carry both ways out, or a model can only guess: the + // preset ids are behind agent_list and the legacy profiles are a closed set. + assert.match(refusal, /a user-approved preset id from agent_list/); + assert.match(refusal, /profile to one of: [^.]*\blocal_read\b/); + }); + + test('rejects an item that sets both selectors, and says which mistake it is', async () => { + let starts = 0; + const runtime = buildRuntime(async () => { + starts += 1; + return childResult(starts); + }); + + const result = await executeTool( + runtime, + buildAgentSwarmTool(), + { + items: [ + { + item_id: 'both-selectors', + task: 'Inspect runtime validation.', + profile: LOCAL_READ_AGENT_PROFILE, + subagent_id: 'reviewer', + }, + ], + }, + new AbortController(), ); + + assert.equal(starts, 0); + const refusal = String((result as { error?: unknown }).error); + // The opposite mistake must not read as the same sentence — a model told + // "neither is set" while it set both learns nothing it can act on. + assert.match(refusal, /subagent_id and profile are both set/); + assert.doesNotMatch(refusal, /Neither subagent_id nor profile is set/); + assert.match(refusal, /a user-approved preset id from agent_list/); }); test('accepts prompt_template with string items and rejects ambiguous template input', () => { diff --git a/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts b/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts new file mode 100644 index 0000000000..4c489be5eb --- /dev/null +++ b/packages/runtime/src/__tests__/pre-dispatch-refusal-ledger.test.ts @@ -0,0 +1,446 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + scanToolLedger, + type LlmConnection, + type RuntimeEvent, + type SessionEvent, + type SessionHeader, +} from '@maka/core'; +import { z } from 'zod'; + +import { createSessionEventMapMemory, mapSessionEventToRuntimeEvent } from '../ai-sdk-flow.js'; +import type { InvocationContext } from '../invocation-context.js'; +import type { RuntimeCommitSink } from '../runtime-commit-sink.js'; +import { LOOP_GATE_IDENTICAL_THRESHOLD, type MakaTool, type ToolRuntime } from '../tool-runtime.js'; +import { createTestToolRuntime } from './execution-boundary-test-helpers.js'; + +/** + * A pre-dispatch refusal must leave the ledger a matched call/response pair. + * + * The lane is the whole point. A call tagged with an `operationId` is claimed by + * the T1 dispatch protocol, so AgentRun skips its generic projection and waits + * for `commitToolPrepared` to persist it — which a refusal never reaches. + * Tagging the call while putting the refusal on the generic lane produced an + * `orphan_response`; the ledger refused that append, the append threw, and a + * recoverable refusal killed the whole turn (issue #2234). These tests assemble + * the ledger the way production does — the commit sink's events plus the mapped + * queue events minus the ones AgentRun skips — and assert the scanner is clean. + */ + +const SESSION_ID = 'session-1'; +const RUN_ID = 'run-1'; +const INVOCATION_ID = 'invocation-1'; +const TURN_ID = 'turn-1'; + +/** `agent_swarm`'s shape: per-item cross-field rule, which is what DeepSeek tripped. */ +const swarmLikeSchema = z + .object({ + items: z.array( + z + .object({ + item_id: z.string(), + task: z.string(), + profile: z.string().optional(), + subagent_id: z.string().optional(), + }) + .superRefine((item, ctx) => { + if (Boolean(item.profile) === Boolean(item.subagent_id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Provide exactly one of subagent_id or legacy profile.', + }); + } + }), + ), + }) + .strict(); + +interface LedgerHarness { + events: SessionEvent[]; + committed: RuntimeEvent[]; + sink: RuntimeCommitSink; +} + +function harness(): LedgerHarness { + const events: SessionEvent[] = []; + const committed: RuntimeEvent[] = []; + let seq = 0; + return { + events, + committed, + sink: { + commitToolPrepared: async (input) => { + committed.push(input.runtimeEvent, input.dispatchRuntimeEvent); + return { created: true, runtimeEventSeq: (seq += 2) }; + }, + commitToolOutcome: async (input) => { + committed.push(input.runtimeEvent); + return { created: true, runtimeEventSeq: (seq += 1) }; + }, + }, + }; +} + +/** + * The ledger as the store would see it: what the commit sink persisted, plus the + * mapped queue events AgentRun does append. `isAtomicToolBoundaryProjection` is + * mirrored here — a protocol-tagged call/response is the sink's to write, so the + * generic lane skips it rather than duplicating the fact. + */ +function projectLedger(h: LedgerHarness): RuntimeEvent[] { + const memory = createSessionEventMapMemory(); + const ctx = { + sessionId: SESSION_ID, + invocationId: INVOCATION_ID, + runId: RUN_ID, + turnId: TURN_ID, + source: 'test', + startedAt: 1, + request: { sessionId: SESSION_ID, turnId: TURN_ID, text: '', source: 'test' }, + newId: () => 'unused', + now: () => 1, + } satisfies InvocationContext; + const generic = h.events + .map((event) => mapSessionEventToRuntimeEvent(event, ctx, memory)) + .filter((event) => { + const atomic = + event.refs?.operationId !== undefined && + (event.content?.kind === 'function_call' || event.content?.kind === 'function_response'); + return !atomic && !event.partial; + }); + return [...h.committed, ...generic]; +} + +function runtimeInput(h: LedgerHarness) { + return { + sessionId: SESSION_ID, + header: header(), + connection: connection(), + modelId: 'mock-model', + runId: RUN_ID, + invocationId: INVOCATION_ID, + runtimeCommitSink: h.sink, + appendMessage: async () => {}, + newId: nextId(), + now: () => 1, + getPermissionPauseTarget: () => null, + }; +} + +function runtimeFor(h: LedgerHarness) { + return createTestToolRuntime(runtimeInput(h)); +} + +function settle( + h: LedgerHarness, + tool: MakaTool, + input: unknown, + options: { runtime?: ToolRuntime; toolCallId?: string; stepId?: string } = {}, +) { + return (options.runtime ?? runtimeFor(h)).settleToolCall({ + tool, + turnId: TURN_ID, + toolCallId: options.toolCallId ?? 'call_00_swarm', + ...(options.stepId ? { stepId: options.stepId } : {}), + input, + abortSignal: new AbortController().signal, + eventSink: { + push: (event) => h.events.push(event), + pushAndWaitUntilConsumed: async (event) => { + h.events.push(event); + }, + }, + }); +} + +/** Every issue the scanner raised, so a failure names what actually broke. */ +function ledgerIssues(h: LedgerHarness): string[] { + const scan = scanToolLedger(projectLedger(h)); + return scan.issues.map((issue) => `${issue.code}@${issue.eventId}`); +} + +const swarmTool: MakaTool = { + name: 'agent_swarm', + description: 'test', + parameters: swarmLikeSchema, + impl: async () => { + assert.fail('invalid arguments must not reach the implementation'); + }, +}; + +/** + * Every pre-dispatch refusal, not just the one that produced the bug report. + * + * These are the paths the old construction-time predicate had to enumerate + * correctly; the push-time model has to keep them on the generic lane by + * construction instead. Driving each one end to end is what turns "the + * enumeration currently agrees with the guards" into something a future edit + * cannot quietly break. + * + * Not covered here: the subagent cap needs five settlements held open + * concurrently, and that interacts with the slot release path rather than the + * lane; `subagent-tool-limit` in the runtime suite owns it. + */ +const REFUSAL_PATHS: Array<{ + name: string; + expect: RegExp; + drive: (h: LedgerHarness) => Promise<{ result: unknown }>; +}> = [ + { + name: 'arguments rejected by the schema', + expect: /arguments failed validation/, + drive: (h) => settle(h, swarmTool, { items: [{ item_id: 'a', task: 'one' }] }), + }, + { + name: 'exclusive-step admission', + expect: /cannot share an assistant step/, + drive: async (h) => { + // agent_swarm is `exclusive_step`: the second call in one step is refused. + const runtime = runtimeFor(h); + const ok = { items: [{ item_id: 'a', task: 'one', subagent_id: 'reviewer' }] }; + const tool: MakaTool = { + ...swarmTool, + executionSemantics: 'exclusive_step', + impl: async () => ({ ok: true }), + }; + await settle(h, tool, ok, { runtime, toolCallId: 'call_first', stepId: 'step-1' }); + return settle(h, tool, ok, { runtime, toolCallId: 'call_second', stepId: 'step-1' }); + }, + }, + { + name: 'loop gate on a repeated identical failing call', + expect: /already failed repeatedly/, + drive: async (h) => { + const runtime = runtimeFor(h); + const tool: MakaTool = { + name: 'Bash', + description: 'test', + parameters: z.object({ command: z.string() }), + impl: async () => { + throw new Error('always fails'); + }, + }; + const input = { command: 'false' }; + let last: { result: unknown } = { result: undefined }; + // The streak is keyed on the args, so distinct call ids still repeat it. + for (let attempt = 0; attempt < LOOP_GATE_IDENTICAL_THRESHOLD; attempt += 1) { + last = await settle(h, tool, input, { runtime, toolCallId: `call_loop_${attempt}` }); + } + return last; + }, + }, + { + name: 'loop gate on a repeated ambiguous Computer Use target', + expect: /ambiguous|same target/i, + drive: async (h) => { + const runtime = runtimeFor(h); + const tool: MakaTool = { + name: 'maka_computer', + description: 'test', + categoryHint: 'computer_use', + parameters: z.object({ + action: z.string(), + app: z.string(), + element_id: z.string(), + }), + // The result shape that arms the ambiguous-target gate. + impl: async () => ({ error: 'stale_frame', failureClass: 'ambiguous_target' }), + }; + const input = { action: 'click_element', app: 'Finder', element_id: '4' }; + await settle(h, tool, input, { runtime, toolCallId: 'call_cu_first' }); + return settle(h, tool, input, { runtime, toolCallId: 'call_cu_second' }); + }, + }, + { + name: 'deferred tool used before its load', + expect: /load_tools/, + drive: async (h) => { + const runtime = runtimeFor(h); + runtime.setGating({ gatedNames: new Set(['Deferred']), activeNames: () => new Set() }); + const tool: MakaTool = { + name: 'Deferred', + description: 'test', + parameters: z.object({}), + impl: async () => ({ ok: true }), + }; + return settle(h, tool, {}, { runtime, toolCallId: 'call_deferred' }); + }, + }, + { + name: 'client-capability boundary read failure', + expect: /boundary read exploded/, + drive: (h) => { + const runtime = createTestToolRuntime({ + ...runtimeInput(h), + readExecutionBoundary: async () => { + throw new Error('boundary read exploded'); + }, + }); + const tool: MakaTool = { + name: 'browser_click', + description: 'test', + categoryHint: 'client_capability', + parameters: z.object({}), + impl: async () => ({ ok: true }), + }; + return settle(h, tool, {}, { runtime, toolCallId: 'call_boundary_read' }); + }, + }, + { + name: 'client-capability blocked by the execution boundary', + expect: /require the Bypass execution boundary/, + drive: (h) => { + // The default test boundary is `external`, not `bypass`. + const tool: MakaTool = { + name: 'browser_click', + description: 'test', + categoryHint: 'client_capability', + parameters: z.object({}), + impl: async () => ({ ok: true }), + }; + return settle(h, tool, {}, { toolCallId: 'call_boundary_blocked' }); + }, + }, +]; + +for (const path of REFUSAL_PATHS) { + test(`refusal keeps the ledger clean: ${path.name}`, async () => { + const h = harness(); + const { result } = await path.drive(h); + + // It is a refusal, and it is the one this row means. + const refusal = (result as { error?: string }).error ?? ''; + assert.notEqual(refusal, '', 'expected a refusal, got a successful result'); + assert.match(refusal, path.expect); + + // And the ledger it leaves behind has no orphan and no lane conflict. + assert.deepEqual(ledgerIssues(h), []); + + // The refused call itself never claims the T1 lane. + const scan = scanToolLedger(projectLedger(h)); + const refused = scan.operations.at(-1); + assert.equal(refused?.dispatchEvent, undefined); + assert.ok(refused?.callEvent, 'the refusal left no call fact at all'); + assert.ok(refused?.responseEvent, 'the refusal left no response fact'); + }); +} + +test('arguments the schema rejects leave a matched call/response pair on the generic lane', async () => { + const h = harness(); + + const { result } = await settle(h, swarmTool, { + items: [ + { item_id: 'a', task: 'one', subagent_id: 'reviewer' }, + { item_id: 'b', task: 'two', subagent_id: 'reviewer' }, + // The third item names neither selector — one slip out of three. + { item_id: 'c', task: 'three' }, + ], + }); + + // The refusal still reaches the model, unchanged. + assert.match( + (result as { error?: string }).error ?? '', + /Tool "agent_swarm" arguments failed validation/, + ); + + // Nothing crossed T1: no prepared/dispatch/outcome commit for a call that + // never ran. + assert.deepEqual(h.committed, []); + + const ledger = projectLedger(h); + const scan = scanToolLedger(ledger); + assert.deepEqual( + scan.issues.map((issue) => issue.code), + [], + `pre-dispatch refusal corrupted the ledger: ${JSON.stringify(scan.issues)}`, + ); + assert.equal(scan.hasCorruption, false); + + // The pair the scanner matched is the refusal's own, and it is untagged — + // both halves on the generic lane. + const [operation, ...extra] = scan.operations; + assert.deepEqual(extra, []); + assert.equal(operation?.toolCallId, 'call_00_swarm'); + assert.equal(operation?.operationId, undefined); + assert.equal(operation?.callEvent?.content?.kind, 'function_call'); + assert.equal(operation?.responseEvent?.content?.kind, 'function_response'); + assert.equal(operation?.dispatchEvent, undefined); +}); + +test('a dispatched call still claims the T1 lane and settles through the commit sink', async () => { + const h = harness(); + const tool: MakaTool = { + name: 'agent_swarm', + description: 'test', + parameters: swarmLikeSchema, + impl: async () => ({ ok: true }), + }; + + const { result } = await settle(h, tool, { + items: [{ item_id: 'a', task: 'one', subagent_id: 'reviewer' }], + }); + assert.deepEqual(result, { ok: true }); + + // Call, dispatch and outcome are the sink's three facts, all under one id. + const operationIds = new Set(h.committed.map((event) => event.refs?.operationId)); + assert.equal(operationIds.size, 1); + const [operationId] = [...operationIds]; + assert.ok(operationId?.startsWith('toolop_'), `unexpected operation id ${operationId}`); + + const scan = scanToolLedger(projectLedger(h)); + assert.deepEqual( + scan.issues.map((issue) => issue.code), + [], + `dispatched call corrupted the ledger: ${JSON.stringify(scan.issues)}`, + ); + const [operation, ...extra] = scan.operations; + assert.deepEqual(extra, []); + assert.equal(operation?.operationId, operationId); + assert.ok(operation?.callEvent, 'the T1 call fact is missing'); + assert.ok(operation?.dispatchEvent, 'the T1 dispatch fact is missing'); + assert.ok(operation?.responseEvent, 'the T1 outcome fact is missing'); +}); + +function nextId(): () => string { + let sequence = 0; + return () => `id-${++sequence}`; +} + +function header(): SessionHeader { + return { + id: SESSION_ID, + workspaceRoot: '/workspace', + cwd: '/workspace', + createdAt: 1, + lastUsedAt: 1, + name: 'Test', + titleIsManual: true, + isFlagged: false, + labels: [], + isArchived: false, + status: 'active', + statusUpdatedAt: 1, + hasUnread: false, + backend: 'ai-sdk', + llmConnectionSlug: 'test', + connectionLocked: true, + model: 'mock-model', + permissionMode: 'bypass', + schemaVersion: 1, + }; +} + +function connection(): LlmConnection { + return { + slug: 'test', + name: 'Test', + providerType: 'openai-compatible', + baseUrl: 'https://example.invalid', + defaultModel: 'mock-model', + enabled: true, + createdAt: 1, + updatedAt: 1, + }; +} diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 889083d3d8..79017dfe80 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -1,7 +1,13 @@ import { describe, test } from 'node:test'; import assert from 'node:assert/strict'; import { setTimeout as timerDelay } from 'node:timers/promises'; -import { deriveTurnRecords, DurableStoreWriteError, isTerminalRuntimeEvent } from '@maka/core'; +import { + deriveTurnRecords, + DurableStoreWriteError, + isTerminalRuntimeEvent, + ToolLedgerCorruptionError, + ToolLedgerRejectionError, +} from '@maka/core'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; import type { AgentRunEvent, @@ -470,6 +476,149 @@ describe('SessionManager terminal ledger invariants', () => { expect(terminals.map((event) => event.id)).toEqual(['terminal-one']); }); + test('a ledger rejection does not cost the run its terminal write', async () => { + // The store latch exists for a store that went away. A rejection means the + // opposite — the ledger is healthy and refused one malformed event — so + // latching only guaranteed that this run could never write its own terminal + // fact: `commitTerminalRun` returns early on an unavailable store, which is + // how a refused tool result left a run reading `running` forever (#2234). + const store = new TinySessionStore(); + // `canonical` is the only durability production ships (SqliteRuntimeStore + // declares it), and it is what makes a rejected append rethrow. A + // best-effort double would swallow the rejection instead and prove a + // weaker thing than production actually does. + const runStore = new TinyAgentRunStore({ + rejectRuntimeEventIds: ['refused-event'], + durability: 'canonical', + }); + const session = await store.create(makeInput()); + const run = new AgentRun({ + sessionId: session.id, + header: session, + userInput: { turnId: 'turn-1', text: 'hello' }, + store, + runStore, + runtimeEventStore: runStore, + newId: nextId(), + now: nextNow(23_000), + hooks: inertAgentRunHooks(store), + }); + await runStore.createRun( + makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), + ); + + // The rejection still fails the caller — a producer bug must not pass + // quietly — and it is recorded on the run. + await assert.rejects( + run.recordRuntimeEvents([ + runtimeEvent({ + id: 'refused-event', + sessionId: session.id, + runId: run.runId, + turnId: run.turnId, + }), + ]), + (error: unknown) => error instanceof ToolLedgerRejectionError, + ); + expect((await runStore.readRun(session.id, run.runId)).traceWriteError).toMatch( + /Tool ledger transition rejected: orphan_response/, + ); + + // The ledger is still open, so the turn can still end. + await run.recordRuntimeEvents([ + runtimeEvent({ + id: 'terminal-after-rejection', + sessionId: session.id, + runId: run.runId, + turnId: run.turnId, + status: 'failed', + actions: { endInvocation: true }, + }), + ]); + + const terminals = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( + isTerminalRuntimeEvent, + ); + expect(terminals.map((event) => event.id)).toEqual(['terminal-after-rejection']); + expect( + (await runStore.readRuntimeEvents(session.id, run.runId)).some( + (event) => event.id === 'refused-event', + ), + ).toBe(false); + }); + + test('an already-corrupt ledger latches the store, and the latch is what costs the terminal fact', async () => { + // The exemption is for a bad candidate against a healthy store; a ledger + // that is already damaged keeps failing closed. This test pins that, and + // pins the price, because the price is not what the fail-closed rationale + // assumes: a corrupt ledger only refuses TOOL facts (production gates the + // health scan behind `isToolLedgerBearingEvent`). The terminal event is not + // one, so the damaged ledger would have accepted it. What actually keeps it + // out is the latch. Assert both halves — the refusal and the collateral — + // so that changing either side has to change this test. + const store = new TinySessionStore(); + const runStore = new TinyAgentRunStore({ corruptLedger: true, durability: 'canonical' }); + const session = await store.create(makeInput()); + const run = new AgentRun({ + sessionId: session.id, + header: session, + userInput: { turnId: 'turn-1', text: 'hello' }, + store, + runStore, + runtimeEventStore: runStore, + newId: nextId(), + now: nextNow(24_000), + hooks: inertAgentRunHooks(store), + }); + await runStore.createRun( + makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), + ); + + // A tool fact is what a damaged ledger refuses. + await assert.rejects( + run.recordRuntimeEvents([ + runtimeEvent({ + id: 'well-formed-tool-fact', + sessionId: session.id, + runId: run.runId, + turnId: run.turnId, + content: { kind: 'function_call', id: 'call-1', name: 'noop', args: {} }, + }), + ]), + (error: unknown) => error instanceof ToolLedgerCorruptionError, + ); + expect((await runStore.readRun(session.id, run.runId)).traceWriteError).toMatch( + /Tool ledger is corrupt: duplicate_call/, + ); + + // The terminal event carries no tool fact, so the corrupt ledger itself + // would take it — the double proves that by not throwing for it. The latch + // is the only thing in its way: under a canonical store `recordRuntimeEvents` + // never reaches the append and replays the latched failure instead. (The + // replay carries the original error object, so the message still reads + // "corrupt" and the stack still points into the double's append — neither + // is evidence the append ran.) + await assert.rejects( + run.recordRuntimeEvents([ + runtimeEvent({ + id: 'terminal-after-corruption', + sessionId: session.id, + runId: run.runId, + turnId: run.turnId, + status: 'failed', + actions: { endInvocation: true }, + }), + ]), + (error: unknown) => error instanceof ToolLedgerCorruptionError, + ); + // This is the assertion that fails if the latch goes away: without it the + // append is attempted, the double accepts a non-tool fact, and the run's + // terminal event lands after all. + expect( + (await runStore.readRuntimeEvents(session.id, run.runId)).some(isTerminalRuntimeEvent), + ).toBe(false); + }); + test('synthetic finalization claims its terminal outcome before its first await', async () => { const headerUpdateStarted = deferred(); const releaseHeaderUpdate = deferred(); @@ -2272,9 +2421,19 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { failTerminalRuntimeEventAppends?: boolean; failTerminalRuntimeEventDurabilityAfterAppend?: boolean; beforeTerminalRuntimeEventAppend?: () => Promise; + /** Event ids the ledger refuses the way a real transition check would. */ + rejectRuntimeEventIds?: readonly string[]; + /** Refuse every append the way an already-corrupt ledger does. */ + corruptLedger?: boolean; + /** SqliteRuntimeStore is `canonical`; declare it when a test needs that shape. */ + durability?: 'best_effort' | 'canonical'; } = {}, ) {} + get durability(): 'best_effort' | 'canonical' | undefined { + return this.options.durability; + } + async createRun(header: AgentRunHeader): Promise { this.headers.set(key(header.sessionId, header.runId), clone(header)); return clone(header); @@ -2325,6 +2484,17 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { if (this.options.failTerminalRuntimeEventAppends && isTerminalRuntimeEvent(event)) { throw new Error('terminal runtime event append failed'); } + if (this.options.corruptLedger && isToolLedgerBearingEvent(event)) { + // Production gates the health scan behind `isToolLedgerBearingEvent` + // (sqlite-runtime-store.ts), so a corrupt ledger refuses tool facts and + // nothing else. A double that refuses EVERY append cannot tell the latch + // apart from the refusal, and the test built on it passes with latching + // deleted outright. + throw new ToolLedgerCorruptionError('duplicate_call', 'some-older-event'); + } + if (this.options.rejectRuntimeEventIds?.includes(event.id)) { + throw new ToolLedgerRejectionError('orphan_response', event.id); + } if (isTerminalRuntimeEvent(event)) await this.options.beforeTerminalRuntimeEventAppend?.(); const eventKey = key(sessionId, runId); this.runtimeEvents.set(eventKey, [...(this.runtimeEvents.get(eventKey) ?? []), clone(event)]); @@ -2447,6 +2617,18 @@ function makeRunHeader(overrides: Partial = {}): AgentRunHeader }; } +/** Mirrors the private predicate in `sqlite-runtime-store.ts` that gates the + * workspace tool-ledger health scan. Kept here so the corrupt-ledger double + * refuses exactly what production refuses, and accepts what it accepts. */ +function isToolLedgerBearingEvent(event: RuntimeEvent): boolean { + return ( + event.content?.kind === 'function_call' || + event.content?.kind === 'function_response' || + event.actions?.toolDispatch !== undefined || + event.actions?.toolRecovery !== undefined + ); +} + function runtimeEvent(overrides: Partial): RuntimeEvent { return { id: 'rt-event', diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index 94070d53ac..65537218ca 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -7,7 +7,12 @@ import type { RuntimeEventStore, ToolBoundaryProtocol, } from '@maka/core'; -import { DurableStoreWriteError, isSessionInlineRun, isTerminalRuntimeEvent } from '@maka/core'; +import { + DurableStoreWriteError, + ToolLedgerRejectionError, + isSessionInlineRun, + isTerminalRuntimeEvent, +} from '@maka/core'; import { Buffer } from 'node:buffer'; import { isDeepStrictEqual } from 'node:util'; import { redactSecrets } from '@maka/core/redaction'; @@ -1647,8 +1652,34 @@ export class AgentRun { ): Promise { if (!this.input.runtimeEventStore || !this.runtimeEventStoreAvailable) return Promise.resolve(); const next = this.runtimeEventQueue.then(operation, operation).catch(async (error) => { - this.runtimeEventStoreAvailable = false; - this.runtimeEventStoreFailure = error; + // A rejection is the ledger refusing one malformed candidate, not the + // store going away: it stays healthy and readable, so the latch would + // only cost this run the writes it still owes — above all its own + // terminal event, which `recordRuntimeEvents` refuses once the store + // reads unavailable. That is how a single refused append left a run at + // `running` with no terminal event and no visible failure (#2234). The + // append still fails the caller (a producer bug must not pass quietly), + // but the ledger stays open so the turn can end the way every other + // failure ends. + // + // Only that one class is exempt. A store that went away keeps latching: + // nothing this run emits next can land. + // + // `ToolLedgerCorruptionError` also keeps latching, but be precise about + // what that buys, because it is less than it looks. A damaged ledger + // refuses TOOL facts only — the health scan sits behind + // `isToolLedgerBearingEvent` — so this run's terminal event, which bears + // no tool fact, is a write the corrupt store would have taken. The latch + // is what keeps it out, and the run ends at `running` with no terminal + // fact: #2234's own shape, for the already-damaged population. Held here + // deliberately rather than fixed in passing — a run that cannot write its + // tool facts should arguably still be allowed to say it ended, but that + // is a behaviour change on a path this commit does not otherwise touch. + // Tracked in #2313; the corrupt-ledger test pins the current price. + if (!(error instanceof ToolLedgerRejectionError)) { + this.runtimeEventStoreAvailable = false; + this.runtimeEventStoreFailure = error; + } await this.enqueueTraceWriteFailure(error, label); if (options.rethrow) throw error; }); diff --git a/packages/runtime/src/agent-swarm-tools.ts b/packages/runtime/src/agent-swarm-tools.ts index fc6b327a67..eabbd8ebda 100644 --- a/packages/runtime/src/agent-swarm-tools.ts +++ b/packages/runtime/src/agent-swarm-tools.ts @@ -472,6 +472,27 @@ function traceAgentSwarm( }); } +/** + * Why a call was refused for its child selector, and what would work instead. + * + * The two failures read very differently to a model and shared one sentence: + * "Provide exactly one of subagent_id or legacy profile." It named neither which + * mistake this was, nor a single value that would have been accepted. A model + * that got two of three items right and slipped on the third was told only that + * — and the field list `formatToolArgsViolationText` appends is the *top-level* + * one, which is not where the violation was (`path: ["items", 2]`). So name the + * mistake, and name both ways out: the preset ids live behind `agent_list`, and + * the legacy profiles are a closed set this schema already knows. + */ +function childSelectorIssueMessage(bothSet: boolean, profiles: readonly string[]): string { + const waysOut = + `Set subagent_id to a user-approved preset id from agent_list, ` + + `or profile to one of: ${profiles.join(', ')}.`; + return bothSet + ? `subagent_id and profile are both set; exactly one of them selects the child. ${waysOut}` + : `Neither subagent_id nor profile is set, so no child is selected. ${waysOut}`; +} + function agentSwarmInputSchema( definitions: readonly AgentDefinition[], profiles: ReturnType, @@ -510,7 +531,7 @@ function agentSwarmInputSchema( if (Boolean(input.profile) === Boolean(input.subagent_id)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'Provide exactly one of subagent_id or legacy profile.', + message: childSelectorIssueMessage(Boolean(input.profile), profiles), }); return; } @@ -651,7 +672,10 @@ function agentSwarmInputSchema( if (Boolean(input.profile) === Boolean(input.subagent_id)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'String items require exactly one of subagent_id or legacy profile.', + message: `String items share one selector for the whole batch. ${childSelectorIssueMessage( + Boolean(input.profile), + profiles, + )}`, }); } if (!input.prompt_template.includes(AGENT_SWARM_PROMPT_TEMPLATE_PLACEHOLDER)) { diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index aa58ab381c..333dd0f985 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -964,57 +964,41 @@ export class ToolRuntime { this.gating !== undefined && this.gating.gatedNames.has(tool.name) && !this.gating.activeNames().has(tool.name); - const rejectedBeforeClientBoundary = - admissionFailure !== undefined || - permissionArgsError !== undefined || - repeatedAmbiguousComputerTarget || - repeatedFailedCall || - deferredToolNotLoaded; - let clientCapabilityBoundary: ExecutionBoundary | undefined; - let clientCapabilityBoundaryReadFailed = false; - let clientCapabilityBoundaryReadError: unknown; - if (!rejectedBeforeClientBoundary && tool.categoryHint === 'client_capability') { - try { - clientCapabilityBoundary = await this.readExecutionBoundary(); - } catch (error) { - clientCapabilityBoundaryReadFailed = true; - clientCapabilityBoundaryReadError = error; - } - } - const clientCapabilityBoundaryRejected = - clientCapabilityBoundaryReadFailed || - (tool.categoryHint === 'client_capability' && clientCapabilityBoundary?.kind !== 'bypass'); - const rejectedBeforeSubagentAdmission = - rejectedBeforeClientBoundary || clientCapabilityBoundaryRejected; - // Slot admission is part of preflight too. Reserve it before assigning a - // durable operation id so a saturated subagent call stays on the generic - // call/response lane just like every other pre-dispatch rejection. - const reservedSubagentSlot = !rejectedBeforeSubagentAdmission && this.reserveSubagentSlot(tool); - const preflightRejected = rejectedBeforeSubagentAdmission || !reservedSubagentSlot; - - // Preflight rejection must remain on the generic call/response lane instead - // of claiming the T1 dispatch protocol. If the call carried an operationId - // here, AgentRun would (correctly) skip its generic projection assuming - // commitToolPrepared already persisted it; the synthetic response would - // then become an orphan. - const operationId = - this.input.runtimeCommitSink && invocationId && !preflightRejected - ? buildToolOperationId({ invocationId, providerToolCallId: toolUseId }) - : undefined; const activityIdentity = { origin: ctx.origin, modelVisibility: ctx.origin === 'code_mode' ? ('hidden' as const) : ('visible' as const), ...(ctx.parentToolCallId ? { parentToolCallId: ctx.parentToolCallId } : {}), ...(ctx.parentOperationId ? { parentOperationId: ctx.parentOperationId } : {}), }; - const startEv: ToolStartEvent = { - type: 'tool_start', - id: operationId ? `${operationId}_call` : this.input.newId(), + // Which lane carries this call's `function_call` fact is not knowable here. + // A pre-dispatch refusal — exclusive-step admission, arguments the schema + // rejects, either loop gate, a deferred tool used before its load, a + // boundary read, the subagent cap — never crosses T1, so its call and its + // synthetic response both belong on the generic call/response lane. Only a + // call that reaches `prepareDurableToolAttempt` may claim the T1 dispatch + // protocol, because only `commitToolPrepared` persists the call under that + // identity; tag a refusal and AgentRun skips the generic projection + // (`isAtomicToolBoundaryProjection`) waiting for a commit that never comes, + // leaving the response an `orphan_response` the ledger refuses — which took + // the whole turn down with it (#2234). + // + // The predecessor of this block answered that by predicting the refusals up + // front: every guard below was hoisted into a `preflightRejected` boolean + // read at construction time. It is correct only while the prediction and + // the guards agree, and nothing holds them together — a new refusal path, + // or a guard that grows a condition its hoisted twin does not, silently + // restores the orphan. Deciding at push time cannot drift, because the + // decision IS the code path taken. + const dispatchOperationId = + this.input.runtimeCommitSink && invocationId + ? buildToolOperationId({ invocationId, providerToolCallId: toolUseId }) + : undefined; + const callEventFacts = { + type: 'tool_start' as const, turnId, ts: now, toolUseId, toolName: tool.name, - ...(operationId ? { operationId } : {}), ...activityIdentity, ...(tool.activityKind ? { activityKind: tool.activityKind } : {}), args: structuredClone(persistedArgs), @@ -1025,6 +1009,39 @@ export class ToolRuntime { ...(toolIntent ? { intent: toolIntent } : {}), ...(stepId !== undefined ? { stepId } : {}), }; + let pushedCallEvent: ToolStartEvent | undefined; + const pushCallEvent = (lane: 'dispatch' | 'preflight'): ToolStartEvent => { + // Idempotent by construction: one call, one call event, whichever lane + // asks for it first. A second ask cannot mint a second id. + if (pushedCallEvent) return pushedCallEvent; + const operationId = lane === 'dispatch' ? dispatchOperationId : undefined; + const event: ToolStartEvent = { + ...callEventFacts, + id: operationId ? `${operationId}_call` : this.input.newId(), + ...(operationId ? { operationId } : {}), + }; + queue.push(event); + pushedCallEvent = event; + return event; + }; + /** + * One pre-dispatch refusal: the call fact on the generic lane, then the + * refusal the model reads, on the same lane. Every refusal below routes + * through here so the pair can never be split across lanes again. + */ + const refuseBeforeDispatch = async (text: string): Promise => { + pushCallEvent('preflight'); + await this.writeSyntheticToolResult( + toolUseId, + turnId, + text, + queue, + undefined, + undefined, + undefined, + activityIdentity, + ); + }; const callMsg: ToolCallMessage = { type: 'tool_call', id: toolUseId, @@ -1043,29 +1060,14 @@ export class ToolRuntime { // timeline and post-restart backfill can pair this call with its step. ...(stepId !== undefined ? { stepId } : {}), }; - try { - await this.input.appendMessage(callMsg); - queue.push(startEv); - trace?.emit('tool', 'tool_started', 'Tool execution started', { - toolUseId, - toolName: tool.name, - ...(tool.categoryHint !== undefined ? { categoryHint: tool.categoryHint } : {}), - }); - } catch (error) { - if (reservedSubagentSlot) this.releaseSubagentSlot(tool); - throw error; - } + await this.input.appendMessage(callMsg); + trace?.emit('tool', 'tool_started', 'Tool execution started', { + toolUseId, + toolName: tool.name, + ...(tool.categoryHint !== undefined ? { categoryHint: tool.categoryHint } : {}), + }); if (admissionFailure) { - await this.writeSyntheticToolResult( - toolUseId, - turnId, - admissionFailure, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(admissionFailure); trace?.emit('tool', 'tool_failed', 'Tool rejected by exclusive-step admission', { toolUseId, toolName: tool.name, @@ -1100,16 +1102,7 @@ export class ToolRuntime { args: executionArgs, error: permissionArgsError, }); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - msg, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(msg); this.input.recordToolInvocation?.({ sessionId: this.input.sessionId, turnId, @@ -1161,16 +1154,7 @@ export class ToolRuntime { // streak stays parked and every further identical repeat stays blocked. if (repeatedAmbiguousComputerTarget) { const reason = formatAmbiguousComputerLoopGateText(); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - reason, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(reason); trace?.emit('tool', 'tool_failed', 'Blocked repeated ambiguous Computer Use target', { toolUseId, toolName: tool.name, @@ -1188,16 +1172,7 @@ export class ToolRuntime { } if (repeatedFailedCall) { const reason = formatLoopGateText(tool.name); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - reason, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(reason); trace?.emit('tool', 'tool_failed', 'Loop-gate blocked a repeated identical failing call', { toolUseId, toolName: tool.name, @@ -1216,16 +1191,7 @@ export class ToolRuntime { // model loads via `load_tools`, then retries next step. if (deferredToolNotLoaded) { const reason = formatDeferredNotLoadedText(tool.name); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - reason, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(reason); trace?.emit('tool', 'tool_failed', 'Deferred tool used before load', { toolUseId, toolName: tool.name, @@ -1236,19 +1202,13 @@ export class ToolRuntime { return this.errorReturn(reason); } + let clientCapabilityBoundary: ExecutionBoundary | undefined; if (tool.categoryHint === 'client_capability') { - if (clientCapabilityBoundaryReadFailed) { - const reason = formatSyntheticToolErrorText(clientCapabilityBoundaryReadError); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - reason, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + try { + clientCapabilityBoundary = await this.readExecutionBoundary(); + } catch (error) { + const reason = formatSyntheticToolErrorText(error); + await refuseBeforeDispatch(reason); trace?.emit('tool', 'tool_failed', 'Client Capability boundary read failed', { toolUseId, toolName: tool.name, @@ -1258,17 +1218,8 @@ export class ToolRuntime { this.recordLoopGateOutcome(callSignature, true); return this.errorReturn(reason); } - if (clientCapabilityBoundary?.kind !== 'bypass') { - await this.writeSyntheticToolResult( - toolUseId, - turnId, - CLIENT_CAPABILITY_BOUNDARY_MESSAGE, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + if (clientCapabilityBoundary.kind !== 'bypass') { + await refuseBeforeDispatch(CLIENT_CAPABILITY_BOUNDARY_MESSAGE); trace?.emit('tool', 'tool_failed', 'Client Capability blocked by execution boundary', { toolUseId, toolName: tool.name, @@ -1280,6 +1231,7 @@ export class ToolRuntime { } } + const reservedSubagentSlot = this.reserveSubagentSlot(tool); if (!reservedSubagentSlot) { trace?.emit('tool', 'tool_failed', 'Tool execution rejected by runtime limit', { toolUseId, @@ -1287,16 +1239,7 @@ export class ToolRuntime { errorClass: 'RuntimeLimit', boundary: 'subagent_tool_admission', }); - await this.writeSyntheticToolResult( - toolUseId, - turnId, - SUBAGENT_TOOL_LIMIT_MESSAGE, - queue, - undefined, - undefined, - undefined, - activityIdentity, - ); + await refuseBeforeDispatch(SUBAGENT_TOOL_LIMIT_MESSAGE); this.recordLoopGateOutcome(callSignature, true); return this.errorReturn(SUBAGENT_TOOL_LIMIT_MESSAGE); } @@ -1305,7 +1248,7 @@ export class ToolRuntime { try { durableAttempt = await this.prepareDurableToolAttempt({ tool, - startEvent: startEv, + startEvent: pushCallEvent('dispatch'), persistedArgs, modelFacingArgs, abortSignal: ctx.abortSignal, @@ -1356,7 +1299,9 @@ export class ToolRuntime { executionBoundary, permissionMode: this.input.header.permissionMode, toolCallId: toolUseId, - ...(operationId ? { operationId } : {}), + // The id the call event actually carries, not the candidate: by here + // `prepareDurableToolAttempt` has pushed it on the dispatch lane. + ...(pushedCallEvent?.operationId ? { operationId: pushedCallEvent.operationId } : {}), abortSignal: ctx.abortSignal, emitOutput: output.emit, ...(trace diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index 94d4a312c5..7d35fb58d7 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -9,6 +9,8 @@ import { canonicalToolArgsHash, createRuntimeBoundaryCursor, runtimePrefixSegment, + ToolLedgerCorruptionError, + ToolLedgerRejectionError, type ContinuationClaimV1, type ImmutableRuntimePrefixV1, type RuntimeEvent, @@ -223,6 +225,80 @@ describe('SqliteRuntimeStore', () => { }); }); + // These two pin the ERROR CLASS, not the message. AgentRun exempts exactly + // one class from the store-unavailable latch (`ToolLedgerRejectionError`), so + // the class is a behavioural contract between storage and runtime — and both + // messages are byte-identical to the plain `Error` strings they replaced, so + // a regression to `throw new Error(...)` would leave every message-matching + // assertion in this suite green while the exemption silently stopped working. + it('rejects an inadmissible candidate with ToolLedgerRejectionError, naming the code', async () => { + await withStore(async (store) => { + // Untagged, so it takes the generic lane — a tagged response is a + // reserved boundary fact and never reaches the transition check. This is + // the exact shape #2234 produced: a result with no call to answer. + const orphan = functionResponseEvent({ + id: 'orphan-response-event', + ts: 11, + refs: { toolCallId: 'provider-call-1' }, + }); + await assert.rejects( + store.appendRuntimeEvent(orphan.sessionId, orphan.runId, orphan), + (error: unknown) => + error instanceof ToolLedgerRejectionError && + error.code === 'orphan_response' && + error.eventId === 'orphan-response-event', + ); + }); + }); + + it('reports pre-existing damage as ToolLedgerCorruptionError, even from another session', async () => { + await withStore(async (store, dbPath) => { + store.close(); + + // Seed damage the store would never have written itself, in a session + // this run never touches: the health scan has no WHERE clause, so one + // damaged operation anywhere in the workspace is what a later append meets. + const raw = new DatabaseSync(dbPath); + const stranded = functionResponseEvent({ + id: 'stranded-response', + sessionId: 'some-other-session', + invocationId: 'some-other-invocation', + runId: 'some-other-run', + ts: 5, + }); + raw + .prepare(` + INSERT INTO runtime_events + (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, + payload_json, committed_at) + VALUES (?, ?, ?, ?, ?, 1, 'function_response', ?, 5) + `) + .run( + stranded.id, + stranded.sessionId, + stranded.invocationId, + stranded.runId, + stranded.turnId, + JSON.stringify(stranded), + ); + raw.close(); + + const reopened = createSqliteRuntimeStore(dbPath); + try { + const healthy = functionCallEvent(); + await assert.rejects( + reopened.appendRuntimeEvent(healthy.sessionId, healthy.runId, healthy), + (error: unknown) => + error instanceof ToolLedgerCorruptionError && + !(error instanceof ToolLedgerRejectionError) && + error.code === 'orphan_response', + ); + } finally { + reopened.close(); + } + }); + }); + it('commits function_call, dispatch fact, and operation projection atomically in T1', async () => { await withStore(async (store) => { const call = functionCallEvent(); diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index 6cf144fcb7..3877b0a6c5 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -19,6 +19,8 @@ import { stableJsonStringify, TOOL_BOUNDARY_PROTOCOL_V1, TOOL_RECOVERY_BUNDLE_CAPABILITY_V1, + ToolLedgerCorruptionError, + ToolLedgerRejectionError, WORKSPACE_AUTHORITY_SESSION_ID, WORKSPACE_VERSION_AUTHORITY_CAPABILITY_V1, validateGenericToolLedgerAppend, @@ -2189,9 +2191,7 @@ export class SqliteRuntimeStore expectedTransition, }); if (!validation.ok) { - throw new Error( - `Tool ledger transition rejected: ${validation.code} at ${validation.eventId}`, - ); + throw new ToolLedgerRejectionError(validation.code, validation.eventId); } } @@ -2203,9 +2203,12 @@ export class SqliteRuntimeStore const health = this.toolLedgerHealth!; if (health.decodeFailure) throw health.decodeFailure.error; if (health.issue) { - throw new Error( - `Tool ledger transition rejected: ${health.issue.code} at ${health.issue.eventId}`, - ); + // Pre-existing damage, not a bad candidate. Note the reach of "refused": + // this gate is only ever consulted for tool-bearing events, so a damaged + // ledger refuses tool facts and takes everything else. Callers that treat + // this as "the store is gone" are overreading it — see the note on the + // latch in `AgentRun.enqueueRuntimeEventStore`. + throw new ToolLedgerCorruptionError(health.issue.code, health.issue.eventId); } }