From 108f07dc4291c66ff24a5601d150d70eaa0277b9 Mon Sep 17 00:00:00 2001 From: qqqys Date: Thu, 10 Sep 2026 21:35:30 +0800 Subject: [PATCH 1/4] fix(goal): persist why a checkpoint failed and show it before the Goal stops A stalled-checkpoint stop always recorded the same reason and advised narrowing the objective, whatever the three failed checks had run into, and nothing showed a failing checkpoint until the Goal stopped. - Keep a capped one-line lastCheckpointFailure on the Goal record: set by every failed check, cleared by a check that succeeds, left alone by a check that proves nothing, and cleared with the stall streak on edit and on the resume of an evidence-limited Goal. - Pick the stall-stop reason from the check that spent the last stall: a full claim list, an unusable verifier answer, or a verifier that never answered. limitKind stays evidence_catalog, so resume is unchanged. - Show the streak and the failure on the Ink and OpenTUI Goal cards, the footer pill, the web shell Goals dialog, and get_goal's lastGoal summary. - Tell the model to retry update_goal right after get_goal when a checkpoint is required, before running other tools. Closes #11326 --- docs/users/features/goals.md | 2 + .../cli/src/ui/components/GoalPill.test.tsx | 34 ++++++ packages/cli/src/ui/components/GoalPill.tsx | 30 +++-- .../messages/GoalStatusMessage.test.tsx | 54 +++++++++ .../components/messages/GoalStatusMessage.tsx | 28 ++++- packages/cli/src/ui/opentui/event-adapter.ts | 2 + .../src/ui/opentui/live-session-model.test.ts | 31 +++++ .../cli/src/ui/opentui/live-session-model.ts | 19 +++ .../cli/src/ui/opentui/transcript-view.tsx | 5 + packages/core/src/goals/goal-protocol.test.ts | 60 ++++++++++ packages/core/src/goals/goal-protocol.ts | 79 ++++++++++++- packages/core/src/goals/goal-reducer.test.ts | 90 ++++++++++++++ packages/core/src/goals/goal-reducer.ts | 9 ++ packages/core/src/goals/goal-runtime.test.ts | 50 +++++++- packages/core/src/goals/goal-runtime.ts | 111 ++++++++++++++++-- packages/core/src/goals/goal-tools.test.ts | 47 ++++++++ packages/core/src/goals/goal-tools.ts | 14 ++- packages/sdk-typescript/src/daemon/index.ts | 1 + packages/sdk-typescript/src/daemon/types.ts | 19 +++ .../components/dialogs/GoalsDialog.test.tsx | 34 ++++++ .../client/components/dialogs/GoalsDialog.tsx | 31 +++++ .../client/daemon/session/mappers.test.ts | 43 +++++++ .../client/daemon/session/mappers.ts | 6 + packages/web-shell/client/i18n.tsx | 8 ++ 24 files changed, 777 insertions(+), 30 deletions(-) diff --git a/docs/users/features/goals.md b/docs/users/features/goals.md index c5f3a7dd767..d6bd4fa9160 100644 --- a/docs/users/features/goals.md +++ b/docs/users/features/goals.md @@ -22,6 +22,8 @@ Each turn the session takes on its own reports what the Goal has spent so far, h A long Goal periodically compresses the evidence it has recorded into checkpoint claims with a side model check, so later turns and the verifier still have it to cite. The check is bounded by [`model.goalCheckpointTimeoutSeconds`](../configuration/settings.md), 180 seconds by default. If its claims overrun the aggregate byte budget, or include a claim over the per-claim character limit, it makes one corrective model call and both calls share that ceiling. A check that does not finish in time is abandoned as inconclusive; it counts toward the checkpoint stall limit only when the evidence window has overflowed, while a non-overflowing check preserves the streak and retries on a later turn. The calls are streamed, so the per-request transport timeout bounds only connect and first response, and the ceiling itself stops at the stream guards' 15-minute lifetime cap because past that the guard, not the setting, ends the check. That 15-minute limit on the setting is fixed, and raising the stream guard's own cap does not lift it. +A failing checkpoint shows up before it stops the Goal. The Goal status card shows how many consecutive checks have stalled out of the three the Goal allows, together with the last failure; the footer pill switches to `checkpoint N/3 stalled`; the web shell's Goals dialog shows the same line; and the model sees both fields when it reads the Goal. A check that fails while the window still has room is shown too, without spending a stall. A Goal stopped by three stalled checkpoints names what the last one ran into. A full claim list that still left evidence behind means the objective produces more evidence than one window holds, so narrow it. An answer that could not be folded into claims means the checkpoint model is not returning the structured output it is asked for, and a check that never answered means the provider was unreachable; narrowing the objective fixes neither. Resuming after any of the three starts a fresh evidence window. + ## Interrupting a Goal Cancelling a Goal turn pauses the Goal. Press Esc while the model is answering or while its tools are still running, and the turn stops, the Goal moves to `paused`, and the card and `/goal` both say why it stopped. Nothing continues until you run `/goal resume`. diff --git a/packages/cli/src/ui/components/GoalPill.test.tsx b/packages/cli/src/ui/components/GoalPill.test.tsx index 5ac0c3264c6..e1b2aee6f7c 100644 --- a/packages/cli/src/ui/components/GoalPill.test.tsx +++ b/packages/cli/src/ui/components/GoalPill.test.tsx @@ -127,6 +127,40 @@ describe('GoalPill', () => { unmount(); }); + it('warns about stalled checkpoints before the stall breaker stops the Goal', () => { + vi.setSystemTime(NOW); + const { lastFrame, unmount } = renderPill({ + snapshot: snapshot('active', 'running', { + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }), + }); + + expect(lastFrame()).toContain('! /goal checkpoint 2/3 stalled'); + // The footer has no room for the failure itself; the status card has it. + expect(lastFrame()).not.toContain('provider failed'); + unmount(); + }); + + it('keeps the plain labels when no checkpoint has stalled', () => { + vi.setSystemTime(NOW); + // A failure on a window with room spends no stall and is the card's to + // show; the footer stays quiet until the streak starts. + const quiet = renderPill({ + snapshot: snapshot('active', 'running', { + lastCheckpointFailure: 'Error: provider failed', + }), + }); + expect(quiet.lastFrame()).toContain('/goal active'); + quiet.unmount(); + + const checking = renderPill({ + snapshot: snapshot('active', 'verifying', { checkpointStalls: 1 }), + }); + expect(checking.lastFrame()).toContain('/goal checking'); + checking.unmount(); + }); + it('adds the current active span to persisted active time', () => { vi.setSystemTime(NOW); const { lastFrame, unmount } = renderPill({ diff --git a/packages/cli/src/ui/components/GoalPill.tsx b/packages/cli/src/ui/components/GoalPill.tsx index d2de46bd5d3..8864b1db799 100644 --- a/packages/cli/src/ui/components/GoalPill.tsx +++ b/packages/cli/src/ui/components/GoalPill.tsx @@ -9,7 +9,10 @@ import { useEffect, useState } from 'react'; import { Text } from 'ink'; import { elapsedActiveTime } from '@qwen-code/qwen-code-core/goals/goal-reducer.js'; import type { Config } from '@qwen-code/qwen-code-core/config/config.js'; -import type { GoalSnapshotV2 } from '@qwen-code/qwen-code-core/goals/goal-protocol.js'; +import { + GOAL_CHECKPOINT_STALL_LIMIT, + type GoalSnapshotV2, +} from '@qwen-code/qwen-code-core/goals/goal-protocol.js'; import type { GoalRuntime } from '@qwen-code/qwen-code-core/goals/goal-runtime.js'; import { useConfig } from '../contexts/ConfigContext.js'; import { theme } from '../semantic-colors.js'; @@ -98,13 +101,24 @@ function presentation(snapshot: GoalSnapshotV2): { if (!goal || goal.status === 'complete') return null; if (goal.status === 'active') { - return snapshot.activity === 'verifying' - ? { - icon: ICON.CIRCLE_EMPTY, - label: 'checking', - color: theme.text.secondary, - } - : { icon: ICON.BULLSEYE, label: 'active', color: theme.text.accent }; + if (snapshot.activity === 'verifying') { + return { + icon: ICON.CIRCLE_EMPTY, + label: 'checking', + color: theme.text.secondary, + }; + } + // A Goal paying a failed checkpoint every turn otherwise looks exactly + // like one that is working, until the stall breaker stops it. + const stalls = goal.checkpointStalls ?? 0; + if (stalls > 0) { + return { + icon: '!', + label: `checkpoint ${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled`, + color: theme.status.warning, + }; + } + return { icon: ICON.BULLSEYE, label: 'active', color: theme.text.accent }; } switch (goal.status) { case 'paused': diff --git a/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx b/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx index 3da8e7c805d..80501cf102d 100644 --- a/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx @@ -187,6 +187,60 @@ describe('', () => { expect(lastFrame()).not.toContain('tokens'); }); + it('shows stalled checkpoints and the last failure on an active card', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain( + 'Checkpoint: 2/3 stalled · Error: provider failed', + ); + }); + + it('shows a checkpoint failure that spent no stall', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain( + 'Checkpoint: last check failed · Error: provider failed', + ); + }); + + it('keeps the failure on the card of a Goal the stall breaker stopped', () => { + // The stop reason names the kind of failure; only this line says which. + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('Reason: checkpoints stalled'); + expect(lastFrame()).toContain( + 'Checkpoint: 3/3 stalled · Error: provider failed', + ); + }); + + it('says nothing about checkpoints on a healthy card', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).not.toContain('Checkpoint'); + }); + it('leaves the legacy card without spend it cannot know', () => { // The legacy props carry an iteration count and nothing else; there is no // record behind them to read a spend off. diff --git a/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx b/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx index fe0862f4ffc..788d8a67bd0 100644 --- a/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx +++ b/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx @@ -6,7 +6,11 @@ import React from 'react'; import { Box, Text } from 'ink'; -import type { GoalSnapshotV2, GoalStateCause } from '@qwen-code/qwen-code-core'; +import { + GOAL_CHECKPOINT_STALL_LIMIT, + type GoalSnapshotV2, + type GoalStateCause, +} from '@qwen-code/qwen-code-core'; import { theme } from '../../semantic-colors.js'; import { ICON } from '../../constants.js'; import { formatDuration } from '../../utils/formatters.js'; @@ -127,6 +131,23 @@ const GoalStateCard: React.FC = ({ goal.status !== 'active' || snapshot.activity === 'verifying' ? goal.lastReason?.trim() : undefined; + // Checkpoint health, shown before the stall breaker has to stop the Goal: + // a Goal paying a failed checkpoint every turn otherwise looks like one + // that is working. A stopped Goal keeps the line, since its stop reason + // names the kind of failure but not the failure itself. + const stalls = goal.checkpointStalls ?? 0; + const checkpointFailure = goal.lastCheckpointFailure?.trim(); + const checkpoint = + goal.status === 'complete' || (stalls === 0 && !checkpointFailure) + ? undefined + : [ + stalls > 0 + ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` + : 'last check failed', + checkpointFailure, + ] + .filter(Boolean) + .join(' · '); return ( @@ -153,6 +174,11 @@ const GoalStateCard: React.FC = ({ Reason: {reason} ) : null} + {checkpoint ? ( + + Checkpoint: {checkpoint} + + ) : null} ); diff --git a/packages/cli/src/ui/opentui/event-adapter.ts b/packages/cli/src/ui/opentui/event-adapter.ts index d36dfde8d0c..19848724e2f 100644 --- a/packages/cli/src/ui/opentui/event-adapter.ts +++ b/packages/cli/src/ui/opentui/event-adapter.ts @@ -790,6 +790,8 @@ export type GoalSnapshotLike = { activeTimeMs?: number; tokensUsed?: number; tokenBudget?: number; + checkpointStalls?: number; + lastCheckpointFailure?: string; lastReason?: string; } | null; activity?: string; diff --git a/packages/cli/src/ui/opentui/live-session-model.test.ts b/packages/cli/src/ui/opentui/live-session-model.test.ts index 19af542a727..a97db8d7453 100644 --- a/packages/cli/src/ui/opentui/live-session-model.test.ts +++ b/packages/cli/src/ui/opentui/live-session-model.test.ts @@ -743,6 +743,37 @@ describe('describeGoalCard (ink GoalStateCard)', () => { }); }); + it('shows checkpoint health, matching the ink card', () => { + expect( + describeGoalCard( + snap({ + objective: 'o', + status: 'active', + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }), + ), + ).toMatchObject({ + checkpoint: 'Checkpoint: 2/3 stalled · Error: provider failed', + }); + expect( + describeGoalCard( + snap({ + objective: 'o', + status: 'active', + lastCheckpointFailure: 'Error: provider failed', + }), + ), + ).toMatchObject({ + checkpoint: 'Checkpoint: last check failed · Error: provider failed', + }); + const healthy = describeGoalCard( + snap({ objective: 'o', status: 'active' }), + ); + expect(healthy).toMatchObject({ state: 'card' }); + expect(healthy).not.toHaveProperty('checkpoint'); + }); + it('builds the subtitle from turns and active time', () => { expect( describeGoalCard( diff --git a/packages/cli/src/ui/opentui/live-session-model.ts b/packages/cli/src/ui/opentui/live-session-model.ts index 8de24ded703..5d63466c8ce 100644 --- a/packages/cli/src/ui/opentui/live-session-model.ts +++ b/packages/cli/src/ui/opentui/live-session-model.ts @@ -16,6 +16,7 @@ import type { HistoryItem } from '../model/streaming-model.js'; import type { GoalSnapshotLike, OpenTuiStreamEvent } from './event-adapter.js'; import type { TodoItem } from '../components/TodoDisplay.js'; import type { AnsiToken } from '@qwen-code/qwen-code-core'; +import { GOAL_CHECKPOINT_STALL_LIMIT } from '@qwen-code/qwen-code-core/goals/goal-protocol.js'; import type { ArenaAgentCardData, CompressionProps } from '../types.js'; import { ICON } from '../constants.js'; import { formatDuration } from '../utils/formatters.js'; @@ -652,6 +653,8 @@ export type GoalCardView = subtitle: string | null; objective: string; reason?: string; + /** Checkpoint stall streak and last failure, when either is set. */ + checkpoint?: string; }; /** Computes the GoalStateCard view (icon/title/subtitle/objective/reason) @@ -726,6 +729,21 @@ export function describeGoalCard( (goal.status ?? 'active') !== 'active' || activity === 'verifying' ? goal.lastReason?.trim() : undefined; + // Checkpoint health, matching the ink card: shown before the stall breaker + // stops the Goal, and kept on the card of a Goal it stopped. + const stalls = goal.checkpointStalls ?? 0; + const checkpointFailure = goal.lastCheckpointFailure?.trim(); + const checkpoint = + goal.status === 'complete' || (stalls === 0 && !checkpointFailure) + ? undefined + : `Checkpoint: ${[ + stalls > 0 + ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` + : 'last check failed', + checkpointFailure, + ] + .filter(Boolean) + .join(' · ')}`; return { state: 'card', icon: lifecycle.icon, @@ -734,6 +752,7 @@ export function describeGoalCard( subtitle: stats.length > 0 ? stats.join(' · ') : null, objective: goal.objective ?? '', reason, + ...(checkpoint ? { checkpoint } : {}), }; } diff --git a/packages/cli/src/ui/opentui/transcript-view.tsx b/packages/cli/src/ui/opentui/transcript-view.tsx index f19e774ea7e..112e3e8e184 100644 --- a/packages/cli/src/ui/opentui/transcript-view.tsx +++ b/packages/cli/src/ui/opentui/transcript-view.tsx @@ -537,6 +537,11 @@ function GoalCard({ {` ${sanitizeTerminalText(view.reason)}`} ) : null} + {view.checkpoint ? ( + + {` ${sanitizeTerminalText(view.checkpoint)}`} + + ) : null} ); } diff --git a/packages/core/src/goals/goal-protocol.test.ts b/packages/core/src/goals/goal-protocol.test.ts index faef9c5e88f..bc1e9066111 100644 --- a/packages/core/src/goals/goal-protocol.test.ts +++ b/packages/core/src/goals/goal-protocol.test.ts @@ -6,6 +6,13 @@ import { describe, expect, it } from 'vitest'; import { + capGoalCheckpointFailure, + GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS, + GOAL_CHECKPOINT_STALLED_REASON, + GOAL_CHECKPOINT_UNREACHABLE_REASON, + GOAL_CHECKPOINT_UNUSABLE_REASON, + goalCheckpointStalledReason, + goalLimitKindForReason, GOAL_PAUSE_REASON_COMMAND, GOAL_PAUSE_REASON_HEADLESS_RUN_ENDED, GOAL_PAUSE_REASON_MAX_CHARACTERS, @@ -115,3 +122,56 @@ describe('goal pause reasons', () => { expect(goalPauseReasonForRunBudget('tool-calls')).toContain('tool-calls'); }); }); + +describe('goal checkpoint stall reasons', () => { + it('advises by what the check that spent the last stall ran into', () => { + expect(goalCheckpointStalledReason('full_claims')).toBe( + GOAL_CHECKPOINT_STALLED_REASON, + ); + expect(goalCheckpointStalledReason('unusable')).toBe( + GOAL_CHECKPOINT_UNUSABLE_REASON, + ); + expect(goalCheckpointStalledReason('unreachable')).toBe( + GOAL_CHECKPOINT_UNREACHABLE_REASON, + ); + // Only the compaction shape is fixed by a narrower objective; telling a + // user whose provider was down to rewrite their Goal is the bug. + expect(GOAL_CHECKPOINT_STALLED_REASON).toContain('narrower objective'); + for (const reason of [ + GOAL_CHECKPOINT_UNUSABLE_REASON, + GOAL_CHECKPOINT_UNREACHABLE_REASON, + ]) { + expect(reason).toContain('Narrowing the objective does not fix this'); + } + }); + + it('keeps resumability on limitKind rather than on the stop prose', () => { + // The stall stop writes `limitKind: 'evidence_catalog'` beside every one + // of these reasons; none may start denoting a kind of its own, or the + // prose and the field could disagree about how a resume behaves. + for (const reason of [ + GOAL_CHECKPOINT_STALLED_REASON, + GOAL_CHECKPOINT_UNUSABLE_REASON, + GOAL_CHECKPOINT_UNREACHABLE_REASON, + ]) { + expect(goalLimitKindForReason(reason)).toBeUndefined(); + } + }); + + it('caps a failure diagnostic on a code point boundary', () => { + expect(capGoalCheckpointFailure(' Error: provider failed ')).toBe( + 'Error: provider failed', + ); + const exact = 'x'.repeat(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS); + expect(capGoalCheckpointFailure(exact)).toBe(exact); + + // Astral characters are two UTF-16 units: a slice by `.length` would + // split one and leave a lone surrogate in the journaled record. + const capped = capGoalCheckpointFailure( + '😀'.repeat(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS + 10), + ); + expect(codePoints(capped)).toBe(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS); + expect(capped.endsWith('…')).toBe(true); + expect([...capped].slice(0, -1).every((char) => char === '😀')).toBe(true); + }); +}); diff --git a/packages/core/src/goals/goal-protocol.ts b/packages/core/src/goals/goal-protocol.ts index 3492fcee892..b1814141163 100644 --- a/packages/core/src/goals/goal-protocol.ts +++ b/packages/core/src/goals/goal-protocol.ts @@ -29,8 +29,74 @@ export const GOAL_CHECKPOINT_STALL_LIMIT = 3; * the loop. */ export const GOAL_NO_PROGRESS_TURN_LIMIT = 3; +/** + * The stall stop for a Goal whose last stalled check folded the window into a + * full claim list and still left evidence behind: compaction itself cannot + * keep up, so the objective is producing more evidence than one window holds. + */ export const GOAL_CHECKPOINT_STALLED_REASON = - 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and each check either came back with a full claim list, came back with a result that could not be folded into claims, or did not come back at all, so every turn paid a checkpoint call and lost uncatalogued evidence. Automatic retries cannot recover. Edit or replace the Goal with a narrower objective before resuming it.'; + 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check folded it into a full claim list that still left evidence behind, so every turn paid a checkpoint call and lost uncatalogued evidence. Automatic retries cannot recover. Edit or replace the Goal with a narrower objective before resuming it.'; + +/** + * The stall stop for a Goal whose last stalled check answered with something + * that could not be folded into claims. The objective may not be too wide at + * all -- the checkpoint model is returning output the runtime cannot accept -- + * so the full-claim-list advice to narrow it would send the user to rewrite a + * Goal that was never the problem. + */ +export const GOAL_CHECKPOINT_UNUSABLE_REASON = + 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check answered with output that could not be folded into claims. Narrowing the objective does not fix this. Check that the checkpoint model returns the structured JSON it is asked for, or switch models, then resume the Goal; resuming starts a fresh evidence window.'; + +/** + * The stall stop for a Goal whose last stalled check failed before the + * checkpoint verifier answered: a timeout, a provider error, a rate limit. + */ +export const GOAL_CHECKPOINT_UNREACHABLE_REASON = + 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check failed before the checkpoint verifier answered. Narrowing the objective does not fix this. Resume the Goal once the provider is reachable; resuming starts a fresh evidence window.'; + +/** + * What the last stalled checkpoint check ran into, which decides the advice + * the stop carries. `full_claims`: the check folded the window into a full + * claim list and still left evidence behind. `unusable`: it answered, but not + * with claims the runtime could accept. `unreachable`: it never answered. + */ +export type GoalCheckpointFailureShape = + | 'full_claims' + | 'unusable' + | 'unreachable'; + +/** The `lastReason` a checkpoint stall stop records for its last failure. */ +export function goalCheckpointStalledReason( + shape: GoalCheckpointFailureShape, +): string { + switch (shape) { + case 'full_claims': + return GOAL_CHECKPOINT_STALLED_REASON; + case 'unusable': + return GOAL_CHECKPOINT_UNUSABLE_REASON; + case 'unreachable': + return GOAL_CHECKPOINT_UNREACHABLE_REASON; + default: { + const exhaustive: never = shape; + return exhaustive; + } + } +} + +/** + * Longest `lastCheckpointFailure` a record keeps. A provider error can carry a + * whole response body, and the record is journaled on every checkpoint check. + */ +export const GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS = 500; + +/** Trims a checkpoint failure diagnostic to the record's bound, by code point. */ +export function capGoalCheckpointFailure(text: string): string { + const trimmed = text.trim(); + const codePoints = [...trimmed]; + return codePoints.length <= GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS + ? trimmed + : `${codePoints.slice(0, GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS - 1).join('')}…`; +} /** * Default autonomous spend window armed on a newly created Goal, in model @@ -206,6 +272,17 @@ export interface GoalRecord { * an evidence-limited Goal. */ checkpointStalls?: number; + /** + * What the most recent failed checkpoint check ran into, as a one-line + * diagnostic (`ErrorName: message`, or the runtime's own phrase for a full + * claim list), capped at GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS. Set by + * every check that fails, whether or not it spends a stall, and kept on the + * record the stall breaker stops, so the stop can be diagnosed from the + * record alone. Cleared by a check that succeeds and by every control + * action that clears `checkpointStalls`; a check that proves nothing either + * way (a turn that recorded no evidence) leaves it as it was. + */ + lastCheckpointFailure?: string; /** * Consecutive autonomous turns that recorded neither a tool result nor a * terminal proposal. A model that only restates status never reaches the diff --git a/packages/core/src/goals/goal-reducer.test.ts b/packages/core/src/goals/goal-reducer.test.ts index 736ba74b9c8..c536150ad83 100644 --- a/packages/core/src/goals/goal-reducer.test.ts +++ b/packages/core/src/goals/goal-reducer.test.ts @@ -864,6 +864,96 @@ describe('goal reducer', () => { expect(edited?.checkpointStalls).toBeUndefined(); }); + it('restores a persisted checkpoint failure and rejects a malformed one', () => { + const stalled = snapshot( + goalRecord({ + checkpointStalls: 1, + lastCheckpointFailure: 'Error: provider failed', + }), + ); + expect(parseGoalSnapshotV2(stalled)).toEqual(stalled); + // A check that failed while the window had room reports its failure + // without spending a stall, so the diagnostic stands on its own. + const unstalled = snapshot( + goalRecord({ lastCheckpointFailure: 'Error: provider failed' }), + ); + expect(parseGoalSnapshotV2(unstalled)).toEqual(unstalled); + expect( + parseGoalSnapshotV2(snapshot(goalRecord({ lastCheckpointFailure: '' }))), + ).toBeUndefined(); + expect( + parseGoalSnapshotV2( + snapshot({ + ...goalRecord(), + lastCheckpointFailure: 42, + } as unknown as GoalRecord), + ), + ).toBeUndefined(); + }); + + it('clears the checkpoint failure wherever it clears the stall streak', () => { + const failure = 'Error: provider failed'; + const resume = { + action: 'resume' as const, + expectedGoalId: 'g-1', + expectedRevision: 1, + }; + const edited = reduceGoalControl( + goalRecord({ checkpointStalls: 2, lastCheckpointFailure: failure }), + { + request: { + action: 'edit', + objective: 'deliver the rest', + expectedGoalId: 'g-1', + expectedRevision: 1, + }, + now: 200, + nextGoalId: 'g-next', + cursor: { recordId: 'r-200' }, + }, + ); + expect(edited?.lastCheckpointFailure).toBeUndefined(); + + const restarted = reduceGoalControl( + goalRecord({ + status: 'usage_limited', + limitKind: 'evidence_catalog', + checkpointStalls: 3, + lastCheckpointFailure: failure, + }), + { + request: resume, + now: 200, + nextGoalId: 'unused', + cursor: { recordId: 'r-200' }, + }, + ); + expect(restarted).toMatchObject({ status: 'active' }); + expect(restarted?.checkpointStalls).toBeUndefined(); + expect(restarted?.lastCheckpointFailure).toBeUndefined(); + + // A paused Goal resumes into the window it left: like the streak, the + // diagnostic is still the truth about that window. + const unpaused = reduceGoalControl( + goalRecord({ + status: 'paused', + checkpointStalls: 2, + lastCheckpointFailure: failure, + }), + { + request: resume, + now: 200, + nextGoalId: 'unused', + cursor: { recordId: 'r-200' }, + }, + ); + expect(unpaused).toMatchObject({ + status: 'active', + checkpointStalls: 2, + lastCheckpointFailure: failure, + }); + }); + it('rejects a snapshot carrying negative spend', () => { expect( parseGoalSnapshotV2(snapshot(goalRecord({ tokensUsed: -1 }))), diff --git a/packages/core/src/goals/goal-reducer.ts b/packages/core/src/goals/goal-reducer.ts index 331084f384c..eec0192a0bb 100644 --- a/packages/core/src/goals/goal-reducer.ts +++ b/packages/core/src/goals/goal-reducer.ts @@ -131,6 +131,7 @@ export function reduceGoalControl( evidenceCursor: copyCursor(transition.cursor), evidenceCheckpoint: undefined, checkpointStalls: undefined, + lastCheckpointFailure: undefined, noProgressTurns: undefined, ...rearmedTokenBudget(current, transition.tokenBudgetGrant), lastReason: undefined, @@ -208,6 +209,7 @@ export function reduceGoalControl( // a different one, so carrying it over would spend the new window's // allowance on the old window's failures. checkpointStalls: undefined, + lastCheckpointFailure: undefined, noProgressTurns: undefined, ...rearmedTokenBudget(current, transition.tokenBudgetGrant), lastReason: undefined, @@ -600,6 +602,7 @@ function parseGoalRecord(value: unknown): GoalRecord | undefined { 'updatedAt', 'evidenceCheckpoint', 'checkpointStalls', + 'lastCheckpointFailure', 'noProgressTurns', 'lastReason', 'limitKind', @@ -626,6 +629,9 @@ function parseGoalRecord(value: unknown): GoalRecord | undefined { !isGoalEvidenceCheckpoint(value['evidenceCheckpoint']) || (value['checkpointStalls'] !== undefined && !isNonNegativeInteger(value['checkpointStalls'])) || + (value['lastCheckpointFailure'] !== undefined && + (typeof value['lastCheckpointFailure'] !== 'string' || + !value['lastCheckpointFailure'])) || (value['noProgressTurns'] !== undefined && !isNonNegativeInteger(value['noProgressTurns'])) || (value['lastReason'] !== undefined && @@ -671,6 +677,9 @@ function parseGoalRecord(value: unknown): GoalRecord | undefined { ...(value['checkpointStalls'] ? { checkpointStalls: value['checkpointStalls'] } : {}), + ...(value['lastCheckpointFailure'] === undefined + ? {} + : { lastCheckpointFailure: value['lastCheckpointFailure'] }), ...(value['noProgressTurns'] ? { noProgressTurns: value['noProgressTurns'] } : {}), diff --git a/packages/core/src/goals/goal-runtime.test.ts b/packages/core/src/goals/goal-runtime.test.ts index 44a28346583..f8c233bb983 100644 --- a/packages/core/src/goals/goal-runtime.test.ts +++ b/packages/core/src/goals/goal-runtime.test.ts @@ -13,6 +13,8 @@ import { GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON, GOAL_CHECKPOINT_STALL_LIMIT, GOAL_CHECKPOINT_STALLED_REASON, + GOAL_CHECKPOINT_UNREACHABLE_REASON, + GOAL_CHECKPOINT_UNUSABLE_REASON, GOAL_DEFAULT_TOKEN_BUDGET, GOAL_NO_PROGRESS_TURN_LIMIT, GOAL_PAUSE_REASON_NO_PROGRESS, @@ -2033,10 +2035,11 @@ describe('goal runtime', () => { `window-${stall}`, ); // Each stalled checkpoint is still written -- the streak is counted on - // the record, not held back in memory. + // the record, not held back in memory, and so is what it ran into. expect(runtime.getSnapshot().goal).toMatchObject({ status: 'active', checkpointStalls: stall, + lastCheckpointFailure: expect.stringContaining('full claim list'), }); } expect(host.started).toHaveLength(GOAL_CHECKPOINT_STALL_LIMIT); @@ -2103,6 +2106,10 @@ describe('goal runtime', () => { expect(checkpointVerifier).toHaveBeenCalledTimes(3); expect(runtime.getSnapshot().goal?.status).toBe('active'); expect(runtime.getSnapshot().goal).not.toHaveProperty('checkpointStalls'); + // A check that succeeded is the only thing that retires the diagnostic. + expect(runtime.getSnapshot().goal).not.toHaveProperty( + 'lastCheckpointFailure', + ); // The streak restarts from zero rather than continuing from two. await runCheckpointTurn(runtime, host, setRecords, records, 101, 'd'); @@ -2175,13 +2182,17 @@ describe('goal runtime', () => { expect(checkpointVerifier).toHaveBeenCalledTimes( GOAL_CHECKPOINT_STALL_LIMIT, ); + // The stop follows the check that spent the last stall. The first two + // came back full, but the last never answered, so the advice is not to + // narrow the objective -- and the record says what the failure was. expect(runtime.getSnapshot()).toMatchObject({ activity: 'idle', goal: { status: 'usage_limited', limitKind: 'evidence_catalog', - lastReason: GOAL_CHECKPOINT_STALLED_REASON, + lastReason: GOAL_CHECKPOINT_UNREACHABLE_REASON, checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT, + lastCheckpointFailure: 'Error: provider failed', }, }); expect(journal.appended.at(-1)?.cause).toBe('usage_limited'); @@ -2211,9 +2222,13 @@ describe('goal runtime', () => { ]; setRecords(records); await runtime.finishTurn(permit); + // The failure is on the record from the first stall on, so every + // surface can show it long before the breaker has to stop the Goal. expect(runtime.getSnapshot().goal).toMatchObject({ status: 'active', checkpointStalls: turn, + lastCheckpointFailure: + 'Error: Goal checkpoint verifier timed out after 30000ms', evidenceCursor: { recordId: cursor }, }); expect(runtime.getSnapshot().goal).not.toHaveProperty( @@ -2237,8 +2252,10 @@ describe('goal runtime', () => { goal: { status: 'usage_limited', limitKind: 'evidence_catalog', - lastReason: GOAL_CHECKPOINT_STALLED_REASON, + lastReason: GOAL_CHECKPOINT_UNREACHABLE_REASON, checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT, + lastCheckpointFailure: + 'Error: Goal checkpoint verifier timed out after 30000ms', }, }); expect(journal.appended.at(-1)?.cause).toBe('usage_limited'); @@ -2282,6 +2299,9 @@ describe('goal runtime', () => { expect(runtime.getSnapshot().goal).toMatchObject({ status: 'active', checkpointStalls: turn, + lastCheckpointFailure: expect.stringMatching( + /^InvalidGoalCheckpointError: /, + ), }); } } @@ -2289,13 +2309,18 @@ describe('goal runtime', () => { expect(checkpointVerifier).toHaveBeenCalledTimes( GOAL_CHECKPOINT_STALL_LIMIT, ); + // The verifier answered every time, just not with claims: the stop says + // so instead of sending the user to narrow an objective that was fine. expect(runtime.getSnapshot()).toMatchObject({ activity: 'idle', goal: { status: 'usage_limited', limitKind: 'evidence_catalog', - lastReason: GOAL_CHECKPOINT_STALLED_REASON, + lastReason: GOAL_CHECKPOINT_UNUSABLE_REASON, checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT, + lastCheckpointFailure: expect.stringMatching( + /^InvalidGoalCheckpointError: /, + ), }, }); expect(journal.appended.at(-1)?.cause).toBe('usage_limited'); @@ -2353,9 +2378,15 @@ describe('goal runtime', () => { checkpointVerifier.mockRejectedValueOnce(new Error('provider failed')); await runCheckpointTurn(runtime, host, setRecords, records, 60, 'b'); + // The failure spends no stall, but it is still a failure the surfaces + // should show: the record carries it beside the unchanged streak. expect(runtime.getSnapshot()).toMatchObject({ activity: 'running', - goal: { status: 'active', checkpointStalls: 1 }, + goal: { + status: 'active', + checkpointStalls: 1, + lastCheckpointFailure: 'Error: provider failed', + }, }); // The room arm leaves the same trace the truncated arm does: the // discarded error is diagnosable from the first failure, not only once @@ -2390,9 +2421,12 @@ describe('goal runtime', () => { // only. That close proved nothing about room, so it keeps the streak. await runCheckpointTurn(runtime, host, setRecords, records, 0, 'quiet'); + // The same close proved nothing about the failure either, so the + // diagnostic the stall left stays until a check actually succeeds. expect(runtime.getSnapshot().goal).toMatchObject({ status: 'active', checkpointStalls: 1, + lastCheckpointFailure: expect.stringContaining('full claim list'), }); expect(host.started).toHaveLength(3); }); @@ -4996,11 +5030,14 @@ describe('goal runtime', () => { // stop: the restored Goal keeps the streak it crashed with, and the // continuation the replay mints re-earns any stall as a live turn. expect(checkpointVerifier).toHaveBeenCalledOnce(); + // The exemption covers the streak, not the diagnostic: the replay did + // fail, and the record says so without spending a stall on it. expect(runtime.getSnapshot()).toMatchObject({ activity: 'running', goal: { status: 'active', checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT - 1, + lastCheckpointFailure: 'Error: provider failed', }, }); expect(journal.appended.map((payload) => payload.cause)).toEqual([ @@ -5025,8 +5062,9 @@ describe('goal runtime', () => { goal: { status: 'usage_limited', limitKind: 'evidence_catalog', - lastReason: GOAL_CHECKPOINT_STALLED_REASON, + lastReason: GOAL_CHECKPOINT_UNREACHABLE_REASON, checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT, + lastCheckpointFailure: 'Error: provider failed', }, }); expect(journal.appended.map((payload) => payload.cause)).toEqual([ diff --git a/packages/core/src/goals/goal-runtime.ts b/packages/core/src/goals/goal-runtime.ts index 6b6d7a4d509..2e8dccc3464 100644 --- a/packages/core/src/goals/goal-runtime.ts +++ b/packages/core/src/goals/goal-runtime.ts @@ -16,15 +16,18 @@ import { type GoalEvidenceRecord, } from './goal-evidence.js'; import { + InvalidGoalCheckpointError, isGoalCheckpointStalled, materializeGoalEvidenceCheckpoint, type GoalCheckpointVerifier, } from './goal-checkpoint.js'; import { GoalCheckpointVerifierInputTooLargeError } from './goal-checkpoint-verifier.js'; import { + capGoalCheckpointFailure, + GOAL_CHECKPOINT_CLAIM_LIMIT, GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON, GOAL_CHECKPOINT_STALL_LIMIT, - GOAL_CHECKPOINT_STALLED_REASON, + goalCheckpointStalledReason, GOAL_DEFAULT_TOKEN_BUDGET, GOAL_EVIDENCE_CATALOG_EXHAUSTED_REASON, GOAL_INFEASIBLE_NEXT_STEP, @@ -34,6 +37,7 @@ import { goalTokenBudgetReason, isGoalTokenBudgetSpent, isRepeatedBlockerProposal, + type GoalCheckpointFailureShape, type GoalControlRequest, type GoalEvidenceCheckpoint, type GoalLimitKind, @@ -70,6 +74,46 @@ import { createDebugLogger } from '../utils/debugLogger.js'; const debugLogger = createDebugLogger('GOAL_RUNTIME'); +/** + * What a failed checkpoint check ran into. The shape decides the advice a + * stall stop carries; the detail is the one-line diagnostic the record keeps. + */ +interface CheckpointFailure { + shape: GoalCheckpointFailureShape; + detail: string; +} + +/** + * Classifies a checkpoint check that threw. An `InvalidGoalCheckpointError` + * (its claim-budget and claim-length subclasses included) means the verifier + * answered with something that could not become claims; anything else -- a + * timeout, a provider error, a rate limit -- means no usable answer arrived. + */ +function describeCheckpointFailure(error: unknown): CheckpointFailure { + return { + shape: + error instanceof InvalidGoalCheckpointError ? 'unusable' : 'unreachable', + detail: capGoalCheckpointFailure( + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + ), + }; +} + +/** The stall a checkpoint that came back at the claim ceiling spends. */ +const FULL_CLAIM_LIST_FAILURE: CheckpointFailure = { + shape: 'full_claims', + detail: `checkpoint came back with a full claim list (${GOAL_CHECKPOINT_CLAIM_LIMIT} claims) while the evidence window overflowed`, +}; + +/** + * What a checkpoint check tells the record about checkpoint health: a failure + * to report, `'clear'` after a check that succeeded, or `undefined` when the + * check proved nothing either way and the previous diagnostic stays. + */ +type CheckpointHealthUpdate = CheckpointFailure | 'clear' | undefined; + export const GOAL_RUNTIME_DISPOSED_MESSAGE = 'Goal runtime has been disposed'; export const STALE_GOAL_TURN_MESSAGE = 'Goal turn permit is no longer valid'; @@ -508,12 +552,32 @@ export function createGoalRuntime( }).catch(() => undefined); }; - const withCheckpointStalls = ( + /** + * The Goal record with its checkpoint health brought up to date: the stall + * streak (zero is spelled as no field) and the last failure diagnostic, + * which moves only when the check actually learned something. + */ + const withCheckpointHealth = ( goal: NonNullable, checkpointStalls: number, + update: CheckpointHealthUpdate, ): NonNullable => { - const { checkpointStalls: _previous, ...rest } = goal; - return checkpointStalls > 0 ? { ...rest, checkpointStalls } : rest; + const { + checkpointStalls: _previousStalls, + lastCheckpointFailure: previousFailure, + ...rest + } = goal; + const lastCheckpointFailure = + update === 'clear' + ? undefined + : update === undefined + ? previousFailure + : update.detail; + return { + ...rest, + ...(checkpointStalls > 0 ? { checkpointStalls } : {}), + ...(lastCheckpointFailure ? { lastCheckpointFailure } : {}), + }; }; const assertAvailable = () => { @@ -1032,6 +1096,7 @@ export function createGoalRuntime( const finishCheckpointCheck = async ( attempt: CheckpointAttempt, outcome: 'room' | 'stalled' | 'inconclusive' = 'inconclusive', + failure?: CheckpointFailure, ): Promise => { await enqueue(async () => { if (!isCurrentCheckpointAttempt(attempt) || !snapshot.goal) return; @@ -1047,11 +1112,18 @@ export function createGoalRuntime( : outcome === 'stalled' ? (snapshot.goal.checkpointStalls ?? 0) + 1 : (snapshot.goal.checkpointStalls ?? 0); + // The diagnostic follows what the check learned, not the streak: a + // check that found room clears it, a check that failed reports why + // whether or not it spent a stall, and a check that never ran leaves + // the previous one standing. + const health: CheckpointHealthUpdate = + outcome === 'room' ? 'clear' : failure; if ( await settleIfCheckpointStalled( attempt, snapshot.goal, checkpointStalls, + health, ) ) { return; @@ -1062,7 +1134,7 @@ export function createGoalRuntime( const checkedSnapshot: GoalSnapshotV2 = { v: GOAL_STATE_VERSION, goal: { - ...withCheckpointStalls(snapshot.goal, checkpointStalls), + ...withCheckpointHealth(snapshot.goal, checkpointStalls, health), activeTimeMs: elapsedActiveTime(snapshot.goal, now), updatedAt: now, }, @@ -1108,19 +1180,26 @@ export function createGoalRuntime( /** * Stops the Goal once its stall streak reaches the limit, persisting the - * streak with the stop so the record explains itself. Returns whether the - * attempt was settled. + * streak and the last failure with the stop so the record explains itself. + * The stop reason follows the check that spent the last stall: narrowing + * the objective is the remedy for a full claim list, not for a verifier + * that answered unusably or never answered. Every arm that spends a stall + * reports its failure; a streak reaching the limit without one falls back + * to the compaction reading. Returns whether the attempt was settled. */ const settleIfCheckpointStalled = async ( attempt: CheckpointAttempt, goal: NonNullable, checkpointStalls: number, + health: CheckpointHealthUpdate, ): Promise => { if (checkpointStalls < GOAL_CHECKPOINT_STALL_LIMIT) return false; + const shape: GoalCheckpointFailureShape = + health !== undefined && health !== 'clear' ? health.shape : 'full_claims'; await settleCheckpointFailure( attempt, - withCheckpointStalls(goal, checkpointStalls), - GOAL_CHECKPOINT_STALLED_REASON, + withCheckpointHealth(goal, checkpointStalls, health), + goalCheckpointStalledReason(shape), 'evidence_catalog', ); return true; @@ -1148,6 +1227,9 @@ export function createGoalRuntime( const checkpointStalls = stalled ? (snapshot.goal.checkpointStalls ?? 0) + 1 : 0; + const health: CheckpointHealthUpdate = stalled + ? FULL_CLAIM_LIST_FAILURE + : 'clear'; // A stopped Goal discards the checkpoint it would have written: a // resumed window restarts from a fresh cursor anyway. if ( @@ -1155,6 +1237,7 @@ export function createGoalRuntime( attempt, snapshot.goal, checkpointStalls, + health, ) ) { return; @@ -1165,7 +1248,7 @@ export function createGoalRuntime( const checkpointSnapshot: GoalSnapshotV2 = { v: GOAL_STATE_VERSION, goal: { - ...withCheckpointStalls(snapshot.goal, checkpointStalls), + ...withCheckpointHealth(snapshot.goal, checkpointStalls, health), evidenceCursor: { recordId: attempt.recordUuid }, evidenceCheckpoint: checkpoint, activeTimeMs: elapsedActiveTime(snapshot.goal, now), @@ -1274,6 +1357,10 @@ export function createGoalRuntime( error, `replay=${replay}`, ); + // The trace above is for an investigation; this is what the record + // keeps, on every arm, so the failure is visible before the stall + // breaker stops the Goal and still readable after it has. + const failure = describeCheckpointFailure(error); // A restore replay is exempt: it runs no turn of its own, so a // transient failure at startup must not spend a streak the restored // session never re-earned. The replay mints a continuation whose own @@ -1288,13 +1375,13 @@ export function createGoalRuntime( // overflowing window run a Goal in circles: each turn paid the // call, kept the same cursor, and was told to retry, with nothing // but the token budget left to stop it. - await finishCheckpointCheck(attempt, 'stalled'); + await finishCheckpointCheck(attempt, 'stalled', failure); return; } // A failure while the window still has room must not abort a // healthy Goal: settle the attempt as bookkeeping so the evidence // stays citable and a later turn retries the checkpoint. - await finishCheckpointCheck(attempt); + await finishCheckpointCheck(attempt, 'inconclusive', failure); return; } await recordCheckpoint( diff --git a/packages/core/src/goals/goal-tools.test.ts b/packages/core/src/goals/goal-tools.test.ts index c25968509b7..a5e67fefe59 100644 --- a/packages/core/src/goals/goal-tools.test.ts +++ b/packages/core/src/goals/goal-tools.test.ts @@ -240,6 +240,53 @@ describe('GetGoalTool', () => { ); }); + it('reports checkpoint health in the last Goal summary', async () => { + // A Goal the stall breaker stopped names the kind of failure in + // lastReason; only these two fields say how often and what it was. + const failure = + 'InvalidGoalCheckpointError: Goal checkpoint verifier returned invalid JSON'; + const config = makeConfig({ + getGoalForWorker: vi.fn(), + getSnapshot: () => ({ + v: 2 as const, + activity: 'idle' as const, + goal: { + goalId: 'goal-1', + revision: 3, + objective: 'Ship Goal v3', + status: 'usage_limited' as const, + evidenceCursor: { recordId: 'record-1' }, + turnCount: 5, + activeTimeMs: 10, + tokensUsed: 0, + createdAt: 1, + updatedAt: 2, + checkpointStalls: 3, + lastCheckpointFailure: failure, + lastReason: 'checkpoints stalled', + limitKind: 'evidence_catalog' as const, + }, + }), + }); + + const result = await execute(new GetGoalTool(config)); + + expect(JSON.parse(String(result.llmContent))).toEqual({ + active: false, + lastGoal: { + goalId: 'goal-1', + revision: 3, + status: 'usage_limited', + turnCount: 5, + activeTimeMs: 10, + tokensUsed: 0, + checkpointStalls: 3, + lastCheckpointFailure: failure, + lastReason: 'checkpoints stalled', + }, + }); + }); + it('keeps the objective and the evidence checkpoint behind the permit', async () => { const config = makeConfig({ getGoalForWorker: vi.fn(), diff --git a/packages/core/src/goals/goal-tools.ts b/packages/core/src/goals/goal-tools.ts index 1696a362121..17f4a91d5f4 100644 --- a/packages/core/src/goals/goal-tools.ts +++ b/packages/core/src/goals/goal-tools.ts @@ -90,6 +90,8 @@ type LastGoalSummary = Pick< | 'activeTimeMs' | 'tokensUsed' | 'tokenBudget' + | 'checkpointStalls' + | 'lastCheckpointFailure' | 'lastReason' >; @@ -158,7 +160,7 @@ export class GetGoalTool extends BaseDeclarativeTool< super( GetGoalTool.Name, ToolDisplayNames.GET_GOAL, - `Read the current Goal identity, objective, evidence cursor, and bounded evidence-reference catalog for this permitted Goal turn. The default "summary" view keeps every read small: checkpoint claims are reported as a count (each claim is already an evidenceCatalog entry with its own preview), entries from this turn and checkpoint entries keep full previews, and entries from earlier turns carry previews shortened to ${SUMMARY_PREVIEW_BYTE_LIMIT} bytes. Every entry uuid is present in both views and is valid for update_goal; request view "full" only when a shortened preview is not enough to decide what to cite. Outside a permitted Goal turn it reports "active": false together with "lastGoal", a scalar summary (goalId, revision, status, turnCount, activeTimeMs, tokensUsed, plus tokenBudget and lastReason when recorded) of the session's most recent Goal, so a Goal that has already stopped can still be inspected. It never returns uncited transcript history or changes Goal state. Use the result silently; do not narrate or acknowledge the retrieval to the user.`, + `Read the current Goal identity, objective, evidence cursor, and bounded evidence-reference catalog for this permitted Goal turn. The default "summary" view keeps every read small: checkpoint claims are reported as a count (each claim is already an evidenceCatalog entry with its own preview), entries from this turn and checkpoint entries keep full previews, and entries from earlier turns carry previews shortened to ${SUMMARY_PREVIEW_BYTE_LIMIT} bytes. Every entry uuid is present in both views and is valid for update_goal; request view "full" only when a shortened preview is not enough to decide what to cite. Outside a permitted Goal turn it reports "active": false together with "lastGoal", a scalar summary (goalId, revision, status, turnCount, activeTimeMs, tokensUsed, plus tokenBudget, checkpointStalls, lastCheckpointFailure and lastReason when recorded) of the session's most recent Goal, so a Goal that has already stopped can still be inspected. The Goal record's checkpointStalls counts consecutive evidence checkpoints that failed to relieve an overflowing catalog (the Goal stops at three), and lastCheckpointFailure says what the most recent failed check ran into. It never returns uncited transcript history or changes Goal state. Use the result silently; do not narrate or acknowledge the retrieval to the user.`, Kind.Read, { type: 'object', @@ -219,6 +221,14 @@ export class GetGoalTool extends BaseDeclarativeTool< ...(goal.tokenBudget === undefined ? {} : { tokenBudget: goal.tokenBudget }), + // A Goal the stall breaker stopped names the kind of failure in + // `lastReason`; these two say how often and what exactly it was. + ...(goal.checkpointStalls + ? { checkpointStalls: goal.checkpointStalls } + : {}), + ...(goal.lastCheckpointFailure === undefined + ? {} + : { lastCheckpointFailure: goal.lastCheckpointFailure }), ...(goal.lastReason === undefined ? {} : { lastReason: goal.lastReason }), }; } @@ -344,7 +354,7 @@ class UpdateGoalInvocation extends BaseToolInvocation< goalLifecycleChanged: false, checkpointRequired: true, nextAction: - 'End this turn without user-facing text so the runtime can checkpoint the evidence catalog. In the next Goal turn, call get_goal and retry the terminal proposal with the new evidence UUIDs.', + 'End this turn without user-facing text so the runtime can checkpoint the evidence catalog. In the next Goal turn, call get_goal first and retry the terminal proposal with the UUIDs it returns before running any other tool: every new tool result can push an older entry out of the bounded catalog and invalidate a UUID you meant to cite.', }), returnDisplay: 'Goal evidence reached its bounded catalog; ending the turn to checkpoint before terminal verification.', diff --git a/packages/sdk-typescript/src/daemon/index.ts b/packages/sdk-typescript/src/daemon/index.ts index 91c74cbe789..f8337bd7a1c 100644 --- a/packages/sdk-typescript/src/daemon/index.ts +++ b/packages/sdk-typescript/src/daemon/index.ts @@ -246,6 +246,7 @@ export { DAEMON_APPROVAL_MODES, DAEMON_ERROR_KINDS, DaemonCapabilityMissingError, + GOAL_CHECKPOINT_STALL_LIMIT, GOAL_PAUSE_REASON_COMMAND, isDaemonContentHash, requireWorkspaceCwd, diff --git a/packages/sdk-typescript/src/daemon/types.ts b/packages/sdk-typescript/src/daemon/types.ts index a510b41b2a3..c5d1528f253 100644 --- a/packages/sdk-typescript/src/daemon/types.ts +++ b/packages/sdk-typescript/src/daemon/types.ts @@ -70,6 +70,18 @@ export interface GoalRecord { tokenBudget?: number; createdAt: number; updatedAt: number; + /** + * Consecutive evidence checkpoints that failed to relieve an overflowing + * window; the Goal stops when this reaches three. Absent means zero, which + * is also what an older daemon's snapshot looks like. + */ + checkpointStalls?: number; + /** + * A one-line diagnostic for the most recent checkpoint check that failed, + * kept until a later check succeeds. Absent when the last check did not fail + * or the daemon predates the field. + */ + lastCheckpointFailure?: string; lastReason?: string; limitKind?: GoalLimitKind; } @@ -94,6 +106,13 @@ export interface GoalSnapshotV2 { */ export const GOAL_PAUSE_REASON_COMMAND = 'Paused with /goal pause.'; +/** + * How many consecutive stalled evidence checkpoints stop a Goal, duplicated so + * a client can show `checkpointStalls` against it. It must match + * `GOAL_CHECKPOINT_STALL_LIMIT` in `packages/core/src/goals/goal-protocol.ts`. + */ +export const GOAL_CHECKPOINT_STALL_LIMIT = 3; + export type GoalControlRequest = | { action: 'create'; objective: string } | { diff --git a/packages/web-shell/client/components/dialogs/GoalsDialog.test.tsx b/packages/web-shell/client/components/dialogs/GoalsDialog.test.tsx index e5a2ee60691..70c0ac6878e 100644 --- a/packages/web-shell/client/components/dialogs/GoalsDialog.test.tsx +++ b/packages/web-shell/client/components/dialogs/GoalsDialog.test.tsx @@ -34,6 +34,8 @@ interface MockGoal { tokenBudget?: number; createdAt: number; updatedAt: number; + checkpointStalls?: number; + lastCheckpointFailure?: string; lastReason?: string; limitKind?: 'evidence_catalog' | 'checkpoint_request'; }; @@ -287,6 +289,38 @@ describe('GoalsDialog', () => { expect(resumeButton()).not.toBeNull(); }); + const checkpointLine = () => + document.querySelector('[data-testid="goal-checkpoint"]')?.textContent; + + it('shows stalled checkpoints and the last failure before the Goal stops', async () => { + await mount([ + withSpend({ + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }), + ]); + + expect(checkpointLine()).toBe( + 'Checkpoint: 2/3 checks stalled · Error: provider failed', + ); + }); + + it('shows a checkpoint failure that spent no stall', async () => { + await mount([ + withSpend({ lastCheckpointFailure: 'Error: provider failed' }), + ]); + + expect(checkpointLine()).toBe( + 'Checkpoint: last check failed · Error: provider failed', + ); + }); + + it('shows no checkpoint line for a healthy Goal', async () => { + await mount([baseGoal()]); + + expect(checkpointLine()).toBeUndefined(); + }); + it('renders a goal with its condition, turn count and judge verdict', async () => { await mount([ baseGoal({ iterations: 3, lastReason: 'two tests still fail' }), diff --git a/packages/web-shell/client/components/dialogs/GoalsDialog.tsx b/packages/web-shell/client/components/dialogs/GoalsDialog.tsx index 99051199684..b6199399a4e 100644 --- a/packages/web-shell/client/components/dialogs/GoalsDialog.tsx +++ b/packages/web-shell/client/components/dialogs/GoalsDialog.tsx @@ -5,6 +5,7 @@ */ import { useCallback, useEffect, useRef, useState } from 'react'; +import { GOAL_CHECKPOINT_STALL_LIMIT } from '@qwen-code/sdk/daemon'; import { buildGoalControlRequest } from '../../utils/goalControlRequest'; import { canResumeGoal } from '../../utils/goalGate'; import { @@ -370,6 +371,24 @@ export function GoalsDialog({ // Shared with `GoalStatusStrip` so the two gates cannot drift apart. const canResume = canResumeGoal(goal); const tokenLabel = getGoalTokenLabel(goal, t); + // Checkpoint health, before the stall breaker has to stop the Goal. + const checkpointStalls = goal.checkpointStalls ?? 0; + const checkpointFailure = goal.lastCheckpointFailure?.trim(); + const checkpointLine = + goal.status === 'complete' || + (checkpointStalls === 0 && !checkpointFailure) + ? undefined + : [ + checkpointStalls > 0 + ? t('goal.checkpointStalled', { + count: checkpointStalls, + limit: GOAL_CHECKPOINT_STALL_LIMIT, + }) + : t('goal.checkpointFailed'), + checkpointFailure, + ] + .filter(Boolean) + .join(' · '); return (
@@ -439,6 +458,18 @@ export function GoalsDialog({
)} + {checkpointLine && ( +
+ + {t('goal.checkpoint')}: + {' '} + {checkpointLine} +
+ )} +
{t(`goal.status.${goal.status}`)} diff --git a/packages/web-shell/client/daemon/session/mappers.test.ts b/packages/web-shell/client/daemon/session/mappers.test.ts index 62b2d3c4a91..3c239cb0342 100644 --- a/packages/web-shell/client/daemon/session/mappers.test.ts +++ b/packages/web-shell/client/daemon/session/mappers.test.ts @@ -1015,6 +1015,49 @@ describe('updateConnectionFromDaemonEvent', () => { }); }); + it('carries checkpoint health through from the wire', () => { + // Same pin as limitKind: the field-by-field rebuild must not drop the + // stall streak or the failure the Goals dialog shows before a stop. + const next = applyEvent( + { status: 'connected', workspaceCwd: '/workspace' }, + { + id: 1, + v: 1, + type: 'session_update', + data: { + update: { + sessionUpdate: 'agent_message_chunk', + _meta: { + goalState: { + v: 2, + activity: 'idle', + goal: { + goalId: 'goal-1', + revision: 3, + objective: 'ship it', + status: 'active', + evidenceCursor: { recordId: 'record-1' }, + turnCount: 2, + activeTimeMs: 10, + createdAt: 1, + updatedAt: 2, + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }, + }, + }, + }, + }, + } as DaemonEvent, + ); + + expect(next.goalState?.goal).toMatchObject({ + status: 'active', + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }); + }); + it('drops an unknown limitKind rather than passing it through', () => { const next = applyEvent( { status: 'connected', workspaceCwd: '/workspace' }, diff --git a/packages/web-shell/client/daemon/session/mappers.ts b/packages/web-shell/client/daemon/session/mappers.ts index 959b626fbda..938612836cd 100644 --- a/packages/web-shell/client/daemon/session/mappers.ts +++ b/packages/web-shell/client/daemon/session/mappers.ts @@ -688,6 +688,8 @@ function getGoalState( ) { return undefined; } + const checkpointStalls = getNumber(source, 'checkpointStalls'); + const lastCheckpointFailure = getString(source, 'lastCheckpointFailure'); const lastReason = getString(source, 'lastReason'); const limitKindRaw = getString(source, 'limitKind'); const limitKind = @@ -711,6 +713,10 @@ function getGoalState( ...(tokenBudget !== undefined ? { tokenBudget } : {}), createdAt, updatedAt, + ...(checkpointStalls !== undefined && checkpointStalls > 0 + ? { checkpointStalls } + : {}), + ...(lastCheckpointFailure ? { lastCheckpointFailure } : {}), ...(lastReason ? { lastReason } : {}), ...(limitKind ? { limitKind } : {}), }, diff --git a/packages/web-shell/client/i18n.tsx b/packages/web-shell/client/i18n.tsx index cd8a6af82df..c7da2bd9100 100644 --- a/packages/web-shell/client/i18n.tsx +++ b/packages/web-shell/client/i18n.tsx @@ -2350,6 +2350,10 @@ const EN: Messages = { 'goal.judge': 'Judge', 'goal.label': 'Goal', 'goal.lastCheck': 'Last check', + 'goal.checkpoint': 'Checkpoint', + 'goal.checkpointStalled': (v) => + `${v?.count ?? 0}/${v?.limit ?? 0} checks stalled`, + 'goal.checkpointFailed': 'last check failed', 'goal.notYetMet': 'not yet met', 'goal.set': 'Goal set', 'goal.statusActive': '/goal active', @@ -5847,6 +5851,10 @@ const ZH: Messages = { 'goal.judge': '判断', 'goal.label': '目标', 'goal.lastCheck': '上次检查', + 'goal.checkpoint': '检查点', + 'goal.checkpointStalled': (v) => + `连续 ${v?.count ?? 0}/${v?.limit ?? 0} 次检查停滞`, + 'goal.checkpointFailed': '最近一次检查失败', 'goal.notYetMet': '尚未满足', 'goal.set': '目标已设置', 'goal.statusActive': '/goal 运行中', From a9e428c89d9c84de4b6b6933bd14bd3d19d3cdbf Mon Sep 17 00:00:00 2001 From: qqqys Date: Fri, 11 Sep 2026 10:37:03 +0800 Subject: [PATCH 2/4] fix(goal): address review on checkpoint failure diagnostics - Read claim-budget and claim-length overruns as capacity failures that keep the narrow-the-objective advice, and rename that shape to capacity. - Reword the no-answer stop so it names provider, timeout and check errors instead of blaming the provider, and rethrow the verifier's own timeout when a provider SDK replaces it with a generic abort error. - Write the diagnostic as one display-safe line (control sequences and bidi overrides removed, whitespace collapsed before the cap) and sanitize it again at the Ink card and the web Goals dialog. - Share one visibility rule across every surface and lastGoal: never on a completed Goal, always during a stall streak, and a stall-free failure only while active; scope the diagnostic on the other checkpoint stops. - Show the stall streak in the web Goal status strip, add a tooltip to the dialog row, and move the dialog copy away from "last check". - Pin the SDK stall limit to core, the room-arm clear, the call-site cap, subclass classification and the retry hint in tests; restate the InvalidGoalCheckpointError contract and update the Goal docs. --- docs/users/features/goals.md | 2 +- ...al-checkpoint-stall-limit-wire-key.test.ts | 20 +++ .../messages/GoalStatusMessage.test.tsx | 54 +++++++ .../components/messages/GoalStatusMessage.tsx | 34 +++-- .../src/ui/opentui/live-session-model.test.ts | 22 +++ .../cli/src/ui/opentui/live-session-model.ts | 30 ++-- .../goals/goal-checkpoint-verifier.test.ts | 28 ++++ .../src/goals/goal-checkpoint-verifier.ts | 10 ++ packages/core/src/goals/goal-checkpoint.ts | 13 +- packages/core/src/goals/goal-protocol.test.ts | 103 +++++++++++-- packages/core/src/goals/goal-protocol.ts | 87 ++++++++--- packages/core/src/goals/goal-runtime.test.ts | 139 +++++++++++++++++- packages/core/src/goals/goal-runtime.ts | 49 ++++-- packages/core/src/goals/goal-tools.test.ts | 75 +++++++++- packages/core/src/goals/goal-tools.ts | 21 ++- .../components/GoalStatusStrip.module.css | 5 + .../components/GoalStatusStrip.test.tsx | 22 +++ .../client/components/GoalStatusStrip.tsx | 25 +++- .../components/dialogs/GoalsDialog.test.tsx | 41 +++++- .../client/components/dialogs/GoalsDialog.tsx | 44 ++++-- packages/web-shell/client/i18n.tsx | 4 +- 21 files changed, 724 insertions(+), 104 deletions(-) create mode 100644 packages/cli/src/ui/commands/goal-checkpoint-stall-limit-wire-key.test.ts diff --git a/docs/users/features/goals.md b/docs/users/features/goals.md index 66557b03c86..aee04d7b3fa 100644 --- a/docs/users/features/goals.md +++ b/docs/users/features/goals.md @@ -24,7 +24,7 @@ Each turn the session takes on its own reports what the Goal has spent so far, h A long Goal periodically compresses the evidence it has recorded into checkpoint claims with a side model check, so later turns and the verifier still have it to cite. The check is bounded by [`model.goalCheckpointTimeoutSeconds`](../configuration/settings.md), 180 seconds by default. If its claims overrun the aggregate byte budget, or include a claim over the per-claim character limit, it makes one corrective model call and both calls share that ceiling. A check that does not finish in time is abandoned as inconclusive; it counts toward the checkpoint stall limit only when the evidence window has overflowed, while a non-overflowing check preserves the streak and retries on a later turn. The calls are streamed, so the per-request transport timeout bounds only connect and first response, and the ceiling itself stops at the stream guards' 15-minute lifetime cap because past that the guard, not the setting, ends the check. That 15-minute limit on the setting is fixed, and raising the stream guard's own cap does not lift it. -A failing checkpoint shows up before it stops the Goal. The Goal status card shows how many consecutive checks have stalled out of the three the Goal allows, together with the last failure; the footer pill switches to `checkpoint N/3 stalled`; the web shell's Goals dialog shows the same line; and the model sees both fields when it reads the Goal. A check that fails while the window still has room is shown too, without spending a stall. A Goal stopped by three stalled checkpoints names what the last one ran into. A full claim list that still left evidence behind means the objective produces more evidence than one window holds, so narrow it. An answer that could not be folded into claims means the checkpoint model is not returning the structured output it is asked for, and a check that never answered means the provider was unreachable; narrowing the objective fixes neither. Resuming after any of the three starts a fresh evidence window. +A failing checkpoint shows up before it stops the Goal. While a stall streak runs, the footer pill switches to `checkpoint N/3 stalled` on its own, and the web shell's Goal status strip shows the same count. Whenever a Goal status card is rendered, for example by `/goal` or by a pause, resume or verifier card, it shows how many consecutive checks have stalled out of the three the Goal allows together with the last failure; the web shell's Goals dialog shows the same line, and the model sees both fields when it reads the Goal. A check that fails while the window still has room is shown too, without spending a stall, but only while the Goal is active, and a completed Goal shows no checkpoint line. The failure is kept as a single line with control characters removed. A Goal stopped by three stalled checkpoints names what the last one ran into. A check that could not fit the window within the checkpoint claim bounds, whether a full claim list that still left evidence behind or claims over the size budget, means the objective produces more evidence than one window holds, so narrow it. An answer that could not be folded into claims means the checkpoint model is not returning the structured output it is asked for, and narrowing the objective does not fix that. A check that never answered can mean an unreachable or rate-limited provider, a check that did not finish within `model.goalCheckpointTimeoutSeconds`, or an error in the check itself; the recorded failure says which. Resuming after any of the three starts a fresh evidence window. ## Interrupting a Goal diff --git a/packages/cli/src/ui/commands/goal-checkpoint-stall-limit-wire-key.test.ts b/packages/cli/src/ui/commands/goal-checkpoint-stall-limit-wire-key.test.ts new file mode 100644 index 00000000000..7d751eedd54 --- /dev/null +++ b/packages/cli/src/ui/commands/goal-checkpoint-stall-limit-wire-key.test.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { GOAL_CHECKPOINT_STALL_LIMIT } from '@qwen-code/qwen-code-core'; +import { GOAL_CHECKPOINT_STALL_LIMIT as SDK_GOAL_CHECKPOINT_STALL_LIMIT } from '@qwen-code/sdk/daemon'; + +// The Web Shell renders the checkpoint stall streak against the SDK's +// hand-duplicated copy of this limit, while the runtime stops a Goal at +// Core's. The SDK has no dependency path to Core, so pin the two copies here, +// where both packages are importable: a drift would show users the wrong N/3 +// rather than fail a build. +describe('goal checkpoint stall limit wire contract', () => { + it('is identical across core and the SDK', () => { + expect(SDK_GOAL_CHECKPOINT_STALL_LIMIT).toBe(GOAL_CHECKPOINT_STALL_LIMIT); + }); +}); diff --git a/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx b/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx index 80501cf102d..72be24624d3 100644 --- a/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/GoalStatusMessage.test.tsx @@ -233,6 +233,60 @@ describe('', () => { ); }); + it('hides checkpoint health on a completed Goal that still carries it', () => { + // The terminal snapshot spreads the record and overrides only `status`, + // so a Goal that completed after a failed check journals both fields. + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('Goal complete'); + expect(lastFrame()).not.toContain('Checkpoint'); + }); + + it('hides a stall-free failure once the Goal stops for another reason', () => { + const paused = render( + , + ); + expect(paused.lastFrame()).not.toContain('Checkpoint'); + + // A running streak is still the truth about the window a resume re-enters. + const streak = render( + , + ); + expect(streak.lastFrame()).toContain('Checkpoint: 2/3 stalled'); + }); + + it('never writes control or bidi characters from the diagnostic to the terminal', () => { + const { lastFrame } = render( + , + ); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('Checkpoint: 1/3 stalled'); + expect(frame).not.toContain('\r'); + expect(frame).not.toContain('\u202e'); + }); + it('says nothing about checkpoints on a healthy card', () => { const { lastFrame } = render( , diff --git a/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx b/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx index 788d8a67bd0..2396492cf20 100644 --- a/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx +++ b/packages/cli/src/ui/components/messages/GoalStatusMessage.tsx @@ -8,10 +8,12 @@ import React from 'react'; import { Box, Text } from 'ink'; import { GOAL_CHECKPOINT_STALL_LIMIT, + goalCheckpointHealthVisible, type GoalSnapshotV2, type GoalStateCause, } from '@qwen-code/qwen-code-core'; import { theme } from '../../semantic-colors.js'; +import { sanitizeTerminalText } from '../../utils/textUtils.js'; import { ICON } from '../../constants.js'; import { formatDuration } from '../../utils/formatters.js'; import { formatTokenCount } from '../../statusLinePresets.js'; @@ -133,21 +135,25 @@ const GoalStateCard: React.FC = ({ : undefined; // Checkpoint health, shown before the stall breaker has to stop the Goal: // a Goal paying a failed checkpoint every turn otherwise looks like one - // that is working. A stopped Goal keeps the line, since its stop reason - // names the kind of failure but not the failure itself. + // that is working. A Goal the breaker stopped keeps the line, since its stop + // reason names the kind of failure but not the failure itself; which + // records show it at all is decided once, in goalCheckpointHealthVisible. + // The diagnostic is cleaned where it is written and again here, because + // this renderer writes straight to the terminal. const stalls = goal.checkpointStalls ?? 0; - const checkpointFailure = goal.lastCheckpointFailure?.trim(); - const checkpoint = - goal.status === 'complete' || (stalls === 0 && !checkpointFailure) - ? undefined - : [ - stalls > 0 - ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` - : 'last check failed', - checkpointFailure, - ] - .filter(Boolean) - .join(' · '); + const checkpointFailure = sanitizeTerminalText( + goal.lastCheckpointFailure ?? '', + ).trim(); + const checkpoint = goalCheckpointHealthVisible(goal) + ? [ + stalls > 0 + ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` + : 'last check failed', + checkpointFailure, + ] + .filter(Boolean) + .join(' · ') + : undefined; return ( diff --git a/packages/cli/src/ui/opentui/live-session-model.test.ts b/packages/cli/src/ui/opentui/live-session-model.test.ts index a97db8d7453..5450f412f3f 100644 --- a/packages/cli/src/ui/opentui/live-session-model.test.ts +++ b/packages/cli/src/ui/opentui/live-session-model.test.ts @@ -772,6 +772,28 @@ describe('describeGoalCard (ink GoalStateCard)', () => { ); expect(healthy).toMatchObject({ state: 'card' }); expect(healthy).not.toHaveProperty('checkpoint'); + + // Same visibility rule as the ink card: never on a completed Goal, and a + // stall-free failure only while the Goal is active. + expect( + describeGoalCard( + snap({ + objective: 'o', + status: 'complete', + checkpointStalls: 1, + lastCheckpointFailure: 'Error: provider failed', + }), + ), + ).not.toHaveProperty('checkpoint'); + expect( + describeGoalCard( + snap({ + objective: 'o', + status: 'paused', + lastCheckpointFailure: 'Error: provider failed', + }), + ), + ).not.toHaveProperty('checkpoint'); }); it('builds the subtitle from turns and active time', () => { diff --git a/packages/cli/src/ui/opentui/live-session-model.ts b/packages/cli/src/ui/opentui/live-session-model.ts index 5d63466c8ce..9067a29b9b5 100644 --- a/packages/cli/src/ui/opentui/live-session-model.ts +++ b/packages/cli/src/ui/opentui/live-session-model.ts @@ -16,7 +16,10 @@ import type { HistoryItem } from '../model/streaming-model.js'; import type { GoalSnapshotLike, OpenTuiStreamEvent } from './event-adapter.js'; import type { TodoItem } from '../components/TodoDisplay.js'; import type { AnsiToken } from '@qwen-code/qwen-code-core'; -import { GOAL_CHECKPOINT_STALL_LIMIT } from '@qwen-code/qwen-code-core/goals/goal-protocol.js'; +import { + GOAL_CHECKPOINT_STALL_LIMIT, + goalCheckpointHealthVisible, +} from '@qwen-code/qwen-code-core/goals/goal-protocol.js'; import type { ArenaAgentCardData, CompressionProps } from '../types.js'; import { ICON } from '../constants.js'; import { formatDuration } from '../utils/formatters.js'; @@ -729,21 +732,20 @@ export function describeGoalCard( (goal.status ?? 'active') !== 'active' || activity === 'verifying' ? goal.lastReason?.trim() : undefined; - // Checkpoint health, matching the ink card: shown before the stall breaker - // stops the Goal, and kept on the card of a Goal it stopped. + // Checkpoint health, matching the ink card under the same visibility rule; + // transcript-view sanitizes the line when it renders it. const stalls = goal.checkpointStalls ?? 0; const checkpointFailure = goal.lastCheckpointFailure?.trim(); - const checkpoint = - goal.status === 'complete' || (stalls === 0 && !checkpointFailure) - ? undefined - : `Checkpoint: ${[ - stalls > 0 - ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` - : 'last check failed', - checkpointFailure, - ] - .filter(Boolean) - .join(' · ')}`; + const checkpoint = goalCheckpointHealthVisible(goal) + ? `Checkpoint: ${[ + stalls > 0 + ? `${stalls}/${GOAL_CHECKPOINT_STALL_LIMIT} stalled` + : 'last check failed', + checkpointFailure, + ] + .filter(Boolean) + .join(' · ')}` + : undefined; return { state: 'card', icon: lifecycle.icon, diff --git a/packages/core/src/goals/goal-checkpoint-verifier.test.ts b/packages/core/src/goals/goal-checkpoint-verifier.test.ts index 1c6cacb05a0..e24b4ba2f0c 100644 --- a/packages/core/src/goals/goal-checkpoint-verifier.test.ts +++ b/packages/core/src/goals/goal-checkpoint-verifier.test.ts @@ -515,6 +515,34 @@ describe('createGoalCheckpointVerifier', () => { expect(caller.signal.aborted).toBe(false); }); + it('reports the timeout when the provider answers the abort with its own error', async () => { + // A real provider SDK rejects an aborted request with its own error and + // drops the reason the signal carried, so the Goal record used to say + // "Request was aborted." without saying the check had timed out. + const generateText = vi.fn().mockImplementation( + (request: { abortSignal?: AbortSignal }) => + new Promise((_resolve, reject) => { + request.abortSignal?.addEventListener('abort', () => { + reject(new Error('Request was aborted.')); + }); + }), + ); + const { config } = finishConfig(generateText); + + await expect( + createGoalCheckpointVerifier(config, { timeoutMs: 1 })(input()), + ).rejects.toThrow('Goal checkpoint verifier timed out after 1ms'); + + // The caller's own abort is an interrupt, not a timeout: it keeps the + // error the provider raised. + const caller = new AbortController(); + const interrupted = createGoalCheckpointVerifier(config, { + timeoutMs: 60_000, + })(input(), caller.signal); + caller.abort(new Error('user interrupt')); + await expect(interrupted).rejects.toThrow('Request was aborted.'); + }); + it.each([ ['the configured ceiling', 45_000, { timeoutMs: 45_000 }], ['the built-in default', GOAL_CHECKPOINT_VERIFIER_DEFAULT_TIMEOUT_MS, {}], diff --git a/packages/core/src/goals/goal-checkpoint-verifier.ts b/packages/core/src/goals/goal-checkpoint-verifier.ts index f5a89d9e1db..db7249f0ec0 100644 --- a/packages/core/src/goals/goal-checkpoint-verifier.ts +++ b/packages/core/src/goals/goal-checkpoint-verifier.ts @@ -341,6 +341,16 @@ export function createGoalCheckpointVerifier( retryCause = error; } } + } catch (error) { + // A provider SDK rejects its own aborted request with its own error + // ("Request was aborted.") and drops the reason the signal carried, so + // a check that ran past this ceiling would never say it timed out. The + // caller's abort is left alone: the runtime treats that as an + // interrupt, not a failed check. + if (timeoutController.signal.aborted && !attemptSignal?.aborted) { + throw timeoutController.signal.reason; + } + throw error; } finally { clearTimeout(timer); } diff --git a/packages/core/src/goals/goal-checkpoint.ts b/packages/core/src/goals/goal-checkpoint.ts index 792fe9dd3c8..aa63012e234 100644 --- a/packages/core/src/goals/goal-checkpoint.ts +++ b/packages/core/src/goals/goal-checkpoint.ts @@ -71,9 +71,16 @@ export function isGoalCheckpointStalled( } /** - * An unusable checkpoint verifier result. Nothing in production branches on - * the class -- the stall breaker counts by window state, not error class -- - * so it is a diagnostic carrier: its name and message are what an + * An unusable checkpoint verifier result. + * + * Production branches on this class. The stall breaker still counts by window + * state, but `describeCheckpointFailure` (goal-runtime.ts) picks the stall + * stop's advice from it: this class reads as "the verifier answered, but not + * with usable claims", its claim-budget and claim-length subclasses read as + * capacity failures, and anything outside the hierarchy reads as "no answer + * arrived". Keep provider-side failures -- transport errors, rate limits, + * rejected requests -- out of this hierarchy, or a provider outage would be + * reported as malformed output. Its name and message are also what an * investigation into a stalled Goal gets to see. */ export class InvalidGoalCheckpointError extends Error { diff --git a/packages/core/src/goals/goal-protocol.test.ts b/packages/core/src/goals/goal-protocol.test.ts index 9e714b87eff..bd54ac0f6cf 100644 --- a/packages/core/src/goals/goal-protocol.test.ts +++ b/packages/core/src/goals/goal-protocol.test.ts @@ -11,6 +11,7 @@ import { GOAL_CHECKPOINT_STALLED_REASON, GOAL_CHECKPOINT_UNREACHABLE_REASON, GOAL_CHECKPOINT_UNUSABLE_REASON, + goalCheckpointHealthVisible, goalCheckpointStalledReason, goalLimitKindForReason, GOAL_PAUSE_REASON_COMMAND, @@ -127,7 +128,7 @@ describe('goal pause reasons', () => { describe('goal checkpoint stall reasons', () => { it('advises by what the check that spent the last stall ran into', () => { - expect(goalCheckpointStalledReason('full_claims')).toBe( + expect(goalCheckpointStalledReason('capacity')).toBe( GOAL_CHECKPOINT_STALLED_REASON, ); expect(goalCheckpointStalledReason('unusable')).toBe( @@ -136,15 +137,24 @@ describe('goal checkpoint stall reasons', () => { expect(goalCheckpointStalledReason('unreachable')).toBe( GOAL_CHECKPOINT_UNREACHABLE_REASON, ); - // Only the compaction shape is fixed by a narrower objective; telling a - // user whose provider was down to rewrite their Goal is the bug. + // Capacity is fixed by a narrower objective; malformed output is not, + // and telling that user to rewrite their Goal is the bug. expect(GOAL_CHECKPOINT_STALLED_REASON).toContain('narrower objective'); - for (const reason of [ - GOAL_CHECKPOINT_UNUSABLE_REASON, - GOAL_CHECKPOINT_UNREACHABLE_REASON, - ]) { - expect(reason).toContain('Narrowing the objective does not fix this'); - } + expect(GOAL_CHECKPOINT_UNUSABLE_REASON).toContain( + 'Narrowing the objective does not fix this', + ); + // A check that never answered may have run past its own ceiling on a + // window too large to verify in time, so its advice keeps every remedy + // that can apply instead of blaming the provider. + expect(GOAL_CHECKPOINT_UNREACHABLE_REASON).toContain( + 'model.goalCheckpointTimeoutSeconds', + ); + expect(GOAL_CHECKPOINT_UNREACHABLE_REASON).toContain( + 'narrow the objective', + ); + expect(GOAL_CHECKPOINT_UNREACHABLE_REASON).not.toContain( + 'does not fix this', + ); }); it('keeps resumability on limitKind rather than on the stop prose', () => { @@ -176,6 +186,81 @@ describe('goal checkpoint stall reasons', () => { expect(capped.endsWith('…')).toBe(true); expect([...capped].slice(0, -1).every((char) => char === '😀')).toBe(true); }); + + it('keeps a failure diagnostic on one display-safe line', () => { + expect(capGoalCheckpointFailure('Error: a\nb c')).toBe('Error: a b c'); + expect(capGoalCheckpointFailure('Error: a\r\n b\tc')).toBe( + 'Error: a b c', + ); + // Escape sequences go whole and bidi overrides go entirely, so no + // surface can be repainted or reordered by a provider's error text. + expect( + capGoalCheckpointFailure( + 'Error: \u001b[31mred\u001b[0m rate\u202e limit\u0007', + ), + ).toBe('Error: red rate limit'); + + // Collapsing runs before the cap, so a response body's indentation + // cannot spend the bound. + const capped = capGoalCheckpointFailure( + `Error:${'\n word'.repeat(200)}`, + ); + expect(capped).not.toMatch(/\s{2,}|\n/); + expect(codePoints(capped)).toBe(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS); + }); +}); + +describe('goal checkpoint health visibility', () => { + const failure = 'Error: provider failed'; + + it.each([ + [ + 'an active Goal whose failure spent no stall', + { status: 'active', lastCheckpointFailure: failure }, + true, + ], + [ + 'an active Goal mid-streak', + { status: 'active', checkpointStalls: 2, lastCheckpointFailure: failure }, + true, + ], + [ + 'a Goal the stall breaker stopped', + { + status: 'usage_limited', + checkpointStalls: 3, + lastCheckpointFailure: failure, + }, + true, + ], + [ + 'a paused Goal that keeps its streak', + { status: 'paused', checkpointStalls: 1, lastCheckpointFailure: failure }, + true, + ], + [ + 'a Goal paused for another reason after a stall-free failure', + { status: 'paused', lastCheckpointFailure: failure }, + false, + ], + [ + 'a Goal stopped by another bound after a stall-free failure', + { status: 'usage_limited', lastCheckpointFailure: failure }, + false, + ], + [ + 'a completed Goal that still carries both fields', + { + status: 'complete', + checkpointStalls: 1, + lastCheckpointFailure: failure, + }, + false, + ], + ['a healthy active Goal', { status: 'active' }, false], + ] as const)('%s', (_label, goal, expected) => { + expect(goalCheckpointHealthVisible(goal)).toBe(expected); + }); }); describe('Goal cadence budget reasons', () => { diff --git a/packages/core/src/goals/goal-protocol.ts b/packages/core/src/goals/goal-protocol.ts index 92277f77218..835068aa7d9 100644 --- a/packages/core/src/goals/goal-protocol.ts +++ b/packages/core/src/goals/goal-protocol.ts @@ -4,6 +4,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { + stripDisplayControlChars, + stripTerminalControlSequences, +} from '../utils/terminalSafe.js'; + export const GOAL_STATE_VERSION = 2 as const; export const GOAL_PROPOSAL_REASON_MAX_CHARACTERS = 8_000; export const GOAL_PROPOSAL_REASON_MAX_BYTES = 16_000; @@ -30,38 +35,45 @@ export const GOAL_CHECKPOINT_STALL_LIMIT = 3; */ export const GOAL_NO_PROGRESS_TURN_LIMIT = 3; /** - * The stall stop for a Goal whose last stalled check folded the window into a - * full claim list and still left evidence behind: compaction itself cannot - * keep up, so the objective is producing more evidence than one window holds. + * The stall stop for a Goal whose last stalled check could not fit the window + * inside the checkpoint's claim bounds: a full claim list that still left + * evidence behind, or well-formed claims over the byte or per-claim length + * budget. Compaction itself cannot keep up -- the objective produces more + * evidence than one window holds -- so narrowing it is the remedy. */ export const GOAL_CHECKPOINT_STALLED_REASON = - 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check folded it into a full claim list that still left evidence behind, so every turn paid a checkpoint call and lost uncatalogued evidence. Automatic retries cannot recover. Edit or replace the Goal with a narrower objective before resuming it.'; + 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check could not fit it within the checkpoint claim bounds, so every turn paid a checkpoint call and lost uncatalogued evidence. Automatic retries cannot recover. Edit or replace the Goal with a narrower objective before resuming it.'; /** - * The stall stop for a Goal whose last stalled check answered with something - * that could not be folded into claims. The objective may not be too wide at - * all -- the checkpoint model is returning output the runtime cannot accept -- - * so the full-claim-list advice to narrow it would send the user to rewrite a - * Goal that was never the problem. + * The stall stop for a Goal whose last stalled check answered with output that + * is not usable claims at all. The objective may not be too wide -- the + * checkpoint model is returning output the runtime cannot accept -- so the + * capacity advice to narrow it would send the user to rewrite a Goal that was + * never the problem. */ export const GOAL_CHECKPOINT_UNUSABLE_REASON = 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check answered with output that could not be folded into claims. Narrowing the objective does not fix this. Check that the checkpoint model returns the structured JSON it is asked for, or switch models, then resume the Goal; resuming starts a fresh evidence window.'; /** - * The stall stop for a Goal whose last stalled check failed before the - * checkpoint verifier answered: a timeout, a provider error, a rate limit. + * The stall stop for a Goal whose last stalled check produced no answer to + * judge. The runtime cannot tell why from here: the provider may be + * unreachable or rate-limited, the check may have run past its own ceiling on + * a window too large to verify in time, or the check itself may have failed. + * The recorded failure says which, so this names every remedy that can apply + * rather than blaming the provider. */ export const GOAL_CHECKPOINT_UNREACHABLE_REASON = - 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check failed before the checkpoint verifier answered. Narrowing the objective does not fix this. Resume the Goal once the provider is reachable; resuming starts a fresh evidence window.'; + 'The current Goal revision ran three consecutive evidence checkpoints without relief: the evidence window overflowed every time, and the last check failed before the checkpoint verifier returned an answer. The recorded checkpoint failure says why: an unreachable or rate-limited provider, a check that did not finish within model.goalCheckpointTimeoutSeconds, or an error in the check itself. Fix the provider, raise that timeout, or narrow the objective so the window checkpoints in time, then resume the Goal; resuming starts a fresh evidence window.'; /** * What the last stalled checkpoint check ran into, which decides the advice - * the stop carries. `full_claims`: the check folded the window into a full - * claim list and still left evidence behind. `unusable`: it answered, but not - * with claims the runtime could accept. `unreachable`: it never answered. + * the stop carries. `capacity`: the check could not fit the window within the + * claim bounds (a full claim list that left evidence behind, or claims over + * the byte or length budget). `unusable`: it answered with output that is not + * usable claims. `unreachable`: no answer arrived to judge. */ export type GoalCheckpointFailureShape = - | 'full_claims' + | 'capacity' | 'unusable' | 'unreachable'; @@ -70,7 +82,7 @@ export function goalCheckpointStalledReason( shape: GoalCheckpointFailureShape, ): string { switch (shape) { - case 'full_claims': + case 'capacity': return GOAL_CHECKPOINT_STALLED_REASON; case 'unusable': return GOAL_CHECKPOINT_UNUSABLE_REASON; @@ -89,15 +101,48 @@ export function goalCheckpointStalledReason( */ export const GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS = 500; -/** Trims a checkpoint failure diagnostic to the record's bound, by code point. */ +/** + * Makes a checkpoint failure safe to keep as a one-line diagnostic: terminal + * control sequences and bidi overrides removed, every run of whitespace (line + * breaks included) collapsed to one space, and the result capped by code + * point. Collapsing runs before the cap, so the bound spends its code points + * on the message rather than on a response body's indentation. The value is + * journaled, handed to the model, and rendered on every Goal surface, so it is + * cleaned once where it is written rather than trusted to each reader. + */ export function capGoalCheckpointFailure(text: string): string { - const trimmed = text.trim(); - const codePoints = [...trimmed]; + const oneLine = stripDisplayControlChars(stripTerminalControlSequences(text)) + .replace(/\s+/g, ' ') + .trim(); + const codePoints = [...oneLine]; return codePoints.length <= GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS - ? trimmed + ? oneLine : `${codePoints.slice(0, GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS - 1).join('')}…`; } +/** + * Whether a Goal surface should show checkpoint health, decided once so every + * card and summary agrees. A completed Goal never does: its checkpoints no + * longer matter, and the terminal snapshot keeps whatever the record carried. + * A running stall streak always does, whatever the status, because it is + * still the truth about the evidence window a resume re-enters. A failure that + * spent no stall shows only while the Goal is active: once the Goal stops or + * pauses for another reason, that diagnostic explains nothing about the stop + * and would read as though it did. + */ +export function goalCheckpointHealthVisible(goal: { + status?: string; + checkpointStalls?: number; + lastCheckpointFailure?: string; +}): boolean { + if (goal.status === 'complete') return false; + if ((goal.checkpointStalls ?? 0) > 0) return true; + return ( + (goal.status ?? 'active') === 'active' && + Boolean(goal.lastCheckpointFailure?.trim()) + ); +} + /** * Default autonomous spend window armed on a newly created Goal, in model * tokens on the `tokensUsed` metric (`totalTokenCount` summed per model call, diff --git a/packages/core/src/goals/goal-runtime.test.ts b/packages/core/src/goals/goal-runtime.test.ts index 63e39800b37..034e493978f 100644 --- a/packages/core/src/goals/goal-runtime.test.ts +++ b/packages/core/src/goals/goal-runtime.test.ts @@ -10,6 +10,7 @@ import type { GoalRecoveryRecord } from './goal-persistence.js'; import { GOAL_INFEASIBLE_NEXT_STEP, GOAL_CHECKPOINT_CLAIM_LIMIT, + GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS, GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON, GOAL_CHECKPOINT_STALL_LIMIT, GOAL_CHECKPOINT_STALLED_REASON, @@ -35,12 +36,16 @@ import { type GoalTurnHost, } from './goal-runtime.js'; import { GoalConflictError } from './goal-reducer.js'; -import type { - GoalCheckpointVerificationResult, - GoalCheckpointVerifier, - GoalCheckpointVerifierInput, +import { + InvalidGoalCheckpointError, + type GoalCheckpointVerificationResult, + type GoalCheckpointVerifier, + type GoalCheckpointVerifierInput, } from './goal-checkpoint.js'; -import { GoalCheckpointVerifierInputTooLargeError } from './goal-checkpoint-verifier.js'; +import { + GoalCheckpointClaimBudgetError, + GoalCheckpointVerifierInputTooLargeError, +} from './goal-checkpoint-verifier.js'; import type { GoalVerifier } from './goal-verifier.js'; // Records the GOAL_RUNTIME debug-log calls so tests can assert that a failed @@ -2148,6 +2153,11 @@ describe('goal runtime', () => { expect(checkpointVerifier).toHaveBeenCalledTimes(2); expect(runtime.getSnapshot().goal?.status).toBe('active'); expect(runtime.getSnapshot().goal).not.toHaveProperty('checkpointStalls'); + // The check that found room is the one that proved the window healthy, + // so it also retires the diagnostic the two stalls left behind. + expect(runtime.getSnapshot().goal).not.toHaveProperty( + 'lastCheckpointFailure', + ); }); it('counts a verifier failure on an overflowing window as a stall', async () => { @@ -2330,6 +2340,125 @@ describe('goal runtime', () => { expect(host.started).toHaveLength(GOAL_CHECKPOINT_STALL_LIMIT); }); + it('reads a claim-budget overrun as capacity, not as unusable output', async () => { + // A budget overrun is well-formed JSON that could not fit the window + // within the checkpoint's bounds -- the capacity failure a narrower + // objective fixes -- so its stop keeps that advice rather than telling + // the user to debug JSON that was valid. + const { host, runtime, checkpointVerifier, setRecords } = stallHarness(); + checkpointVerifier.mockRejectedValue( + new GoalCheckpointClaimBudgetError(20_000), + ); + await runtime.dispatch({ action: 'create', objective: 'deliver result' }); + + let records: RuntimeRecord[] = []; + for (let turn = 1; turn <= GOAL_CHECKPOINT_STALL_LIMIT; turn++) { + records = await runCheckpointTurn( + runtime, + host, + setRecords, + records, + 101, + `budget-${turn}`, + ); + } + + expect(runtime.getSnapshot().goal).toMatchObject({ + status: 'usage_limited', + limitKind: 'evidence_catalog', + lastReason: GOAL_CHECKPOINT_STALLED_REASON, + checkpointStalls: GOAL_CHECKPOINT_STALL_LIMIT, + lastCheckpointFailure: expect.stringMatching( + /^GoalCheckpointClaimBudgetError: /, + ), + }); + }); + + it('reads any other subclass of the unusable-result error as unusable output', async () => { + // Pins `instanceof` rather than an exact-constructor check: the hierarchy + // is open, and a new unusable shape must keep the unusable advice. + class ProbeUnusableCheckpointError extends InvalidGoalCheckpointError {} + const { host, runtime, checkpointVerifier, setRecords } = stallHarness(); + checkpointVerifier.mockRejectedValue( + new ProbeUnusableCheckpointError('probe'), + ); + await runtime.dispatch({ action: 'create', objective: 'deliver result' }); + + let records: RuntimeRecord[] = []; + for (let turn = 1; turn <= GOAL_CHECKPOINT_STALL_LIMIT; turn++) { + records = await runCheckpointTurn( + runtime, + host, + setRecords, + records, + 101, + `probe-${turn}`, + ); + } + + expect(runtime.getSnapshot().goal).toMatchObject({ + status: 'usage_limited', + lastReason: GOAL_CHECKPOINT_UNUSABLE_REASON, + }); + }); + + it('records a bounded, one-line, display-safe failure', async () => { + // The only production call site of the record's bound: a provider error + // carrying a whole response body must not reach the journal, the model + // or a card intact. + const { host, runtime, checkpointVerifier, setRecords } = stallHarness(); + checkpointVerifier.mockRejectedValueOnce( + new Error( + `upstream said:\n ${'x'.repeat(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS + 100)}\r\u202e`, + ), + ); + await runtime.dispatch({ action: 'create', objective: 'deliver result' }); + + await runCheckpointTurn(runtime, host, setRecords, [], 101, 'body'); + + const value = runtime.getSnapshot().goal!.lastCheckpointFailure!; + expect([...value]).toHaveLength(GOAL_CHECKPOINT_FAILURE_MAX_CHARACTERS); + expect(value.endsWith('…')).toBe(true); + expect(value.startsWith('Error: upstream said: xxx')).toBe(true); + expect(value).not.toMatch(/[\n\r\u202e]/); + }); + + it('records the failure that made the checkpoint request too large on the stop it caused', async () => { + // The stop's diagnostic belongs to this stop, not to whatever an earlier + // check left: the too-large request is itself the failure to report. + const { host, runtime, checkpointVerifier, setRecords } = stallHarness(); + checkpointVerifier.mockRejectedValueOnce(new Error('provider failed')); + await runtime.dispatch({ action: 'create', objective: 'deliver result' }); + + const records = await runCheckpointTurn( + runtime, + host, + setRecords, + [], + 101, + 'first', + ); + expect(runtime.getSnapshot().goal).toMatchObject({ + checkpointStalls: 1, + lastCheckpointFailure: 'Error: provider failed', + }); + + checkpointVerifier.mockRejectedValueOnce( + new GoalCheckpointVerifierInputTooLargeError(300_000), + ); + await runCheckpointTurn(runtime, host, setRecords, records, 101, 'second'); + + expect(runtime.getSnapshot().goal).toMatchObject({ + status: 'usage_limited', + limitKind: 'checkpoint_request', + lastReason: GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON, + checkpointStalls: 1, + lastCheckpointFailure: expect.stringMatching( + /^GoalCheckpointVerifierInputTooLargeError: /, + ), + }); + }); + it('does not count an unusable result while the window has room', async () => { const { host, runtime, checkpointVerifier, setRecords } = stallHarness(); checkpointVerifier.mockResolvedValue({ claims: [] }); diff --git a/packages/core/src/goals/goal-runtime.ts b/packages/core/src/goals/goal-runtime.ts index 8c41e710b90..e4bb9f983e4 100644 --- a/packages/core/src/goals/goal-runtime.ts +++ b/packages/core/src/goals/goal-runtime.ts @@ -21,7 +21,11 @@ import { materializeGoalEvidenceCheckpoint, type GoalCheckpointVerifier, } from './goal-checkpoint.js'; -import { GoalCheckpointVerifierInputTooLargeError } from './goal-checkpoint-verifier.js'; +import { + GoalCheckpointClaimBudgetError, + GoalCheckpointClaimLengthError, + GoalCheckpointVerifierInputTooLargeError, +} from './goal-checkpoint-verifier.js'; import { capGoalCheckpointFailure, GOAL_CHECKPOINT_CLAIM_LIMIT, @@ -88,15 +92,24 @@ interface CheckpointFailure { } /** - * Classifies a checkpoint check that threw. An `InvalidGoalCheckpointError` - * (its claim-budget and claim-length subclasses included) means the verifier - * answered with something that could not become claims; anything else -- a - * timeout, a provider error, a rate limit -- means no usable answer arrived. + * Classifies a checkpoint check that threw. A claim-budget or claim-length + * overrun is well-formed output that could not fit the window within the + * checkpoint's bounds -- the same capacity failure as a full claim list, with + * the same remedy -- so those two subclasses are tested before their base + * class. Any other `InvalidGoalCheckpointError` means the verifier answered + * with output that is not usable claims. Anything else means no answer + * arrived to judge: a provider error, the check's own timeout, or a failure + * inside the check. */ function describeCheckpointFailure(error: unknown): CheckpointFailure { return { shape: - error instanceof InvalidGoalCheckpointError ? 'unusable' : 'unreachable', + error instanceof GoalCheckpointClaimBudgetError || + error instanceof GoalCheckpointClaimLengthError + ? 'capacity' + : error instanceof InvalidGoalCheckpointError + ? 'unusable' + : 'unreachable', detail: capGoalCheckpointFailure( error instanceof Error ? `${error.name}: ${error.message}` @@ -107,7 +120,7 @@ function describeCheckpointFailure(error: unknown): CheckpointFailure { /** The stall a checkpoint that came back at the claim ceiling spends. */ const FULL_CLAIM_LIST_FAILURE: CheckpointFailure = { - shape: 'full_claims', + shape: 'capacity', detail: `checkpoint came back with a full claim list (${GOAL_CHECKPOINT_CLAIM_LIMIT} claims) while the evidence window overflowed`, }; @@ -1257,7 +1270,7 @@ export function createGoalRuntime( ): Promise => { if (checkpointStalls < GOAL_CHECKPOINT_STALL_LIMIT) return false; const shape: GoalCheckpointFailureShape = - health !== undefined && health !== 'clear' ? health.shape : 'full_claims'; + health !== undefined && health !== 'clear' ? health.shape : 'capacity'; await settleCheckpointFailure( attempt, withCheckpointHealth(goal, checkpointStalls, health), @@ -1267,14 +1280,31 @@ export function createGoalRuntime( return true; }; + /** + * Stops the Goal at a checkpoint bound other than the stall breaker. The + * record's diagnostic is scoped to this stop: an arm whose cause is itself a + * failed check (the request that was too large) records that failure, and + * every other arm clears whatever an earlier, unrelated check left, since + * its own `lastReason` already says what happened. + */ const recordCheckpointFailure = async ( attempt: CheckpointAttempt, reason: string, limitKind?: GoalLimitKind, + health: CheckpointHealthUpdate = 'clear', ): Promise => { await enqueue(async () => { if (!isCurrentCheckpointAttempt(attempt) || !snapshot.goal) return; - await settleCheckpointFailure(attempt, snapshot.goal, reason, limitKind); + await settleCheckpointFailure( + attempt, + withCheckpointHealth( + snapshot.goal, + snapshot.goal.checkpointStalls ?? 0, + health, + ), + reason, + limitKind, + ); }); }; @@ -1410,6 +1440,7 @@ export function createGoalRuntime( attempt, GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON, 'checkpoint_request', + describeCheckpointFailure(error), ); return; } diff --git a/packages/core/src/goals/goal-tools.test.ts b/packages/core/src/goals/goal-tools.test.ts index 6d5f5330b06..4fa106e0272 100644 --- a/packages/core/src/goals/goal-tools.test.ts +++ b/packages/core/src/goals/goal-tools.test.ts @@ -295,6 +295,75 @@ describe('GetGoalTool', () => { }); }); + it('leaves checkpoint health out of the summary of a completed Goal', async () => { + // The terminal snapshot keeps whatever the record carried; a Goal that + // completed cleanly must not be reported with a stale failure. + const config = makeConfig({ + getGoalForWorker: vi.fn(), + getSnapshot: () => ({ + v: 2 as const, + activity: 'idle' as const, + goal: { + goalId: 'goal-1', + revision: 3, + objective: 'Ship Goal v3', + status: 'complete' as const, + evidenceCursor: { recordId: 'record-1' }, + turnCount: 5, + activeTimeMs: 10, + tokensUsed: 0, + createdAt: 1, + updatedAt: 2, + checkpointStalls: 1, + lastCheckpointFailure: 'Error: provider failed', + lastReason: 'Evidence satisfies the objective', + }, + }), + }); + + const { lastGoal } = JSON.parse( + String((await execute(new GetGoalTool(config))).llmContent), + ); + + expect(lastGoal).toMatchObject({ + status: 'complete', + lastReason: 'Evidence satisfies the objective', + }); + expect(lastGoal).not.toHaveProperty('checkpointStalls'); + expect(lastGoal).not.toHaveProperty('lastCheckpointFailure'); + }); + + it('leaves a stall-free failure out of the summary of a Goal paused for another reason', async () => { + const config = makeConfig({ + getGoalForWorker: vi.fn(), + getSnapshot: () => ({ + v: 2 as const, + activity: 'idle' as const, + goal: { + goalId: 'goal-1', + revision: 3, + objective: 'Ship Goal v3', + status: 'paused' as const, + evidenceCursor: { recordId: 'record-1' }, + turnCount: 5, + activeTimeMs: 10, + tokensUsed: 0, + createdAt: 1, + updatedAt: 2, + lastCheckpointFailure: 'Error: provider failed', + lastReason: 'no progress in three turns', + }, + }), + }); + + const { lastGoal } = JSON.parse( + String((await execute(new GetGoalTool(config))).llmContent), + ); + + expect(lastGoal).toMatchObject({ status: 'paused' }); + expect(lastGoal).not.toHaveProperty('lastCheckpointFailure'); + }); + it('keeps the objective and the evidence checkpoint behind the permit', async () => { const config = makeConfig({ getGoalForWorker: vi.fn(), @@ -1039,7 +1108,11 @@ describe('UpdateGoalTool', () => { readyForVerification: false, goalLifecycleChanged: false, checkpointRequired: true, - nextAction: expect.stringContaining('checkpoint the evidence catalog'), + // A retry that runs another tool first can push a cited entry out of + // the catalog, so the hint says to retry before anything else. + nextAction: expect.stringMatching( + /checkpoint the evidence catalog[\s\S]*before running any other tool/, + ), }); expect(result.terminateTurn).toBe(true); expect(recordTerminalProposal).not.toHaveBeenCalled(); diff --git a/packages/core/src/goals/goal-tools.ts b/packages/core/src/goals/goal-tools.ts index f16eae8b67b..fdd6d4088af 100644 --- a/packages/core/src/goals/goal-tools.ts +++ b/packages/core/src/goals/goal-tools.ts @@ -40,6 +40,7 @@ import { goalTurnContext } from './goal-turn-context.js'; import { type GoalBlockerKind, type GoalControlRequest, + goalCheckpointHealthVisible, GOAL_PROPOSAL_REASON_MAX_CHARACTERS, type GoalRecord, type GoalSnapshotV2, @@ -228,13 +229,19 @@ export class GetGoalTool extends BaseDeclarativeTool< ? {} : { activeTimeBudgetMs: goal.activeTimeBudgetMs }), // A Goal the stall breaker stopped names the kind of failure in - // `lastReason`; these two say how often and what exactly it was. - ...(goal.checkpointStalls - ? { checkpointStalls: goal.checkpointStalls } + // `lastReason`; these two say how often and what exactly it was. They + // follow the visibility rule every rendered card uses, so a Goal that + // completed cleanly is not reported with a stale failure. + ...(goalCheckpointHealthVisible(goal) + ? { + ...(goal.checkpointStalls + ? { checkpointStalls: goal.checkpointStalls } + : {}), + ...(goal.lastCheckpointFailure === undefined + ? {} + : { lastCheckpointFailure: goal.lastCheckpointFailure }), + } : {}), - ...(goal.lastCheckpointFailure === undefined - ? {} - : { lastCheckpointFailure: goal.lastCheckpointFailure }), ...(goal.lastReason === undefined ? {} : { lastReason: goal.lastReason }), }; } @@ -422,7 +429,7 @@ export class UpdateGoalTool extends BaseDeclarativeTool< super( UpdateGoalTool.Name, ToolDisplayNames.UPDATE_GOAL, - 'Propose that the current Goal is complete or blocked. Before calling, call get_goal in the current turn and cite only values from evidenceCatalog.entries[].uuid, never goalId, turnId, or lineageTurnIds. If completion depends on user-facing content delivered in the current turn, emit only the content required by the objective, then call get_goal, wait for its result, and call update_goal in a later model step with the returned delivered_output UUID. Do not add progress or completion commentary when the objective requires an exact output format. For blocked proposals, use authority when a user or maintainer decision or permission is required, external when an unavailable external resource or capability is evidenced, repeated for the same evidenced blocker with the exact same reason text across three consecutive Goal turns, and infeasible when a cited external_fact (a tool result, not your own text) shows the objective cannot be satisfied as written -- it contradicts itself, names a target that verifiably does not exist, or needs an action no tool can perform; infeasible is not for difficulty, uncertainty, information you could still obtain, or wanting to ask, and its reason must state what was checked and why no in-scope work could satisfy the objective. Omitting blockerKind follows the repeated-blocker audit. Core records at most one proposal for the exact permitted turn and queues eligible proposals for independent verification. This tool never changes the Goal lifecycle or claims a terminal result. Do not tell the user the Goal is complete or blocked. If this tool reports readyForVerification or checkpointRequired, end the turn without additional user-facing text; after checkpointRequired, call get_goal and retry in the next Goal turn. Otherwise continue the turn without claiming a terminal result. The Goal status card reports the independent verification result.', + 'Propose that the current Goal is complete or blocked. Before calling, call get_goal in the current turn and cite only values from evidenceCatalog.entries[].uuid, never goalId, turnId, or lineageTurnIds. If completion depends on user-facing content delivered in the current turn, emit only the content required by the objective, then call get_goal, wait for its result, and call update_goal in a later model step with the returned delivered_output UUID. Do not add progress or completion commentary when the objective requires an exact output format. For blocked proposals, use authority when a user or maintainer decision or permission is required, external when an unavailable external resource or capability is evidenced, repeated for the same evidenced blocker with the exact same reason text across three consecutive Goal turns, and infeasible when a cited external_fact (a tool result, not your own text) shows the objective cannot be satisfied as written -- it contradicts itself, names a target that verifiably does not exist, or needs an action no tool can perform; infeasible is not for difficulty, uncertainty, information you could still obtain, or wanting to ask, and its reason must state what was checked and why no in-scope work could satisfy the objective. Omitting blockerKind follows the repeated-blocker audit. Core records at most one proposal for the exact permitted turn and queues eligible proposals for independent verification. This tool never changes the Goal lifecycle or claims a terminal result. Do not tell the user the Goal is complete or blocked. If this tool reports readyForVerification or checkpointRequired, end the turn without additional user-facing text; after checkpointRequired, call get_goal first in the next Goal turn and retry before running any other tool, because every new tool result can push a cited entry out of the bounded catalog. Otherwise continue the turn without claiming a terminal result. The Goal status card reports the independent verification result.', Kind.Think, { type: 'object', diff --git a/packages/web-shell/client/components/GoalStatusStrip.module.css b/packages/web-shell/client/components/GoalStatusStrip.module.css index a5923875381..053eb6282c3 100644 --- a/packages/web-shell/client/components/GoalStatusStrip.module.css +++ b/packages/web-shell/client/components/GoalStatusStrip.module.css @@ -60,6 +60,11 @@ color: color-mix(in srgb, var(--muted-foreground) 78%, transparent); } +.checkpoint { + flex: 0 0 auto; + color: var(--warning-color); +} + .actions { display: inline-flex; align-items: center; diff --git a/packages/web-shell/client/components/GoalStatusStrip.test.tsx b/packages/web-shell/client/components/GoalStatusStrip.test.tsx index 9313729a8b7..8830f1cc84d 100644 --- a/packages/web-shell/client/components/GoalStatusStrip.test.tsx +++ b/packages/web-shell/client/components/GoalStatusStrip.test.tsx @@ -244,4 +244,26 @@ describe('GoalStatusStrip', () => { ?.textContent, ).toBe('2.5M / 30.0M tokens'); }); + + it('shows a running checkpoint stall streak, like the terminal footer pill', () => { + render('active', { + checkpointStalls: 2, + lastCheckpointFailure: 'Error: provider failed', + }); + + expect( + container.querySelector('[data-testid="goal-checkpoint-stalls"]') + ?.textContent, + ).toBe('2/3 checks stalled'); + // The failure text belongs to the Goals dialog; the strip has no room. + expect(container.textContent).not.toContain('provider failed'); + }); + + it('shows no streak when no checkpoint has stalled', () => { + render('active', { lastCheckpointFailure: 'Error: provider failed' }); + + expect( + container.querySelector('[data-testid="goal-checkpoint-stalls"]'), + ).toBeNull(); + }); }); diff --git a/packages/web-shell/client/components/GoalStatusStrip.tsx b/packages/web-shell/client/components/GoalStatusStrip.tsx index d65e6ae80d3..133b4d565d2 100644 --- a/packages/web-shell/client/components/GoalStatusStrip.tsx +++ b/packages/web-shell/client/components/GoalStatusStrip.tsx @@ -1,5 +1,8 @@ import { useEffect, useState } from 'react'; -import type { GoalSnapshotV2 } from '@qwen-code/sdk/daemon'; +import { + GOAL_CHECKPOINT_STALL_LIMIT, + type GoalSnapshotV2, +} from '@qwen-code/sdk/daemon'; import { Pause, Pencil, Play, Target, Trash2 } from 'lucide-react'; import { useI18n } from '../i18n'; import { formatRuntime } from '../utils/formatRuntime'; @@ -67,6 +70,10 @@ export function GoalStatusStrip({ const canPause = goal.status === 'active'; const canResume = canResumeGoal(goal); const tokenLabel = getGoalTokenLabel(goal, t); + // Like the terminal footer pill: a running stall streak shows here, where a + // daemon-session user is already looking, while the failure text itself is + // left to the Goals dialog, which has room for it. + const checkpointStalls = goal.checkpointStalls ?? 0; return (
) : null} + {checkpointStalls > 0 ? ( + <> + + + {t('goal.checkpointStalled', { + count: checkpointStalls, + limit: GOAL_CHECKPOINT_STALL_LIMIT, + })} + + + ) : null}