Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion packages/core/src/goals/goal-evidence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ function complete(evidenceRefs: string[]): GoalTerminalProposal {
}

function blocked(
blockerKind: 'authority' | 'external' | 'repeated',
blockerKind: NonNullable<GoalTerminalProposal['blockerKind']>,
evidenceRefs: string[],
): GoalTerminalProposal {
return {
Expand Down Expand Up @@ -1042,6 +1042,59 @@ describe('Goal evidence lineage and blockers', () => {
},
);

it('holds an infeasible blocker to external facts, not user input or prose', () => {
// Infeasibility is a claim about the world. A user can authorise a stop
// (that is `authority`), but cannot make an objective impossible, and
// the assistant saying it is impossible is exactly the exit this kind
// must not become -- so only a tool result can carry it.
const records = [
record('cursor', 'system'),
record('user', 'user', {
provenance: 'real_user',
turnId: 'turn-2',
text: 'Please target the v9 branch',
}),
record('probe', 'tool_result', {
provenance: 'tool_result',
turnId: 'turn-3',
toolResponse: { output: "fatal: branch 'v9' not found" },
}),
record('assistant', 'assistant', {
provenance: 'assistant_output',
turnId: 'turn-3',
text: 'The v9 branch does not exist, so this objective cannot be met.',
}),
];

expect(() =>
validate(records, blocked('infeasible', ['assistant'])),
).toThrowError(
expect.objectContaining({
code: 'infeasible_blocker_external_fact_required',
}),
);
expect(() =>
validate(records, blocked('infeasible', ['user', 'assistant'])),
).toThrowError(
expect.objectContaining({
code: 'infeasible_blocker_external_fact_required',
}),
);
expect(
validate(records, blocked('infeasible', ['probe', 'assistant']))
.citedRecords[0],
).toMatchObject({ uuid: 'probe', proofKind: 'external_fact' });
// Like the other immediate blockers, it cannot leave newer evidence
// uncited: a later record could contradict the impossibility.
expect(() =>
validate(records, blocked('infeasible', ['probe'])),
).toThrowError(
expect.objectContaining({
code: 'immediate_blocker_newer_evidence_required',
}),
);
});

it.each(['authority', 'external'] as const)(
'gates an immediate %s blocker on checkpoint claims like raw evidence',
(blockerKind) => {
Expand Down
18 changes: 17 additions & 1 deletion packages/core/src/goals/goal-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ export type InvalidGoalEvidenceReferenceCode =
| 'wrong_turn_lineage'
| 'catalog_truncated'
| 'immediate_blocker_external_evidence_required'
| 'infeasible_blocker_external_fact_required'
| 'immediate_blocker_newer_evidence_required'
| 'repeated_blocker_turn_coverage';

Expand Down Expand Up @@ -890,9 +891,24 @@ function validateBlockerCoverage(
): void {
if (proposal.status !== 'blocked') return;

// Infeasibility is a claim about the world, so it is held to external
// facts only: user input can authorise stopping, but it cannot make an
// objective impossible, and assistant prose saying so is exactly the
// "I think this can't be done" exit this kind must not become.
if (
proposal.blockerKind === 'infeasible' &&
!citedRecords.some(({ proofKind }) => proofKind === 'external_fact')
) {
throw new InvalidGoalEvidenceReferenceError(
'infeasible_blocker_external_fact_required',
'An infeasible blocker requires cited external tool evidence of the fact that makes the objective unsatisfiable.',
);
}

if (
proposal.blockerKind === 'authority' ||
proposal.blockerKind === 'external'
proposal.blockerKind === 'external' ||
proposal.blockerKind === 'infeasible'
) {
if (
!citedRecords.some(
Expand Down
30 changes: 28 additions & 2 deletions packages/core/src/goals/goal-protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,11 +263,27 @@ export interface GoalStateResponse {
snapshot: GoalSnapshotV2;
}

/**
* Why a Goal is blocked.
*
* `authority` and `external` stop immediately on cited user or external
* evidence. `repeated` (the default) needs the same evidenced blocker on
* three consecutive turns. `infeasible` also stops immediately: the cited
* external fact shows the objective cannot be satisfied as written, so no
* amount of retrying would help -- waiting three turns to say so is the
* runaway this kind exists to end.
*/
export type GoalBlockerKind =
| 'authority'
| 'external'
| 'repeated'
| 'infeasible';

export interface GoalTerminalProposal {
status: 'complete' | 'blocked';
reason: string;
evidenceRefs: string[];
blockerKind?: 'authority' | 'external' | 'repeated';
blockerKind?: GoalBlockerKind;
}

export function isRepeatedBlockerProposal(
Expand All @@ -276,10 +292,20 @@ export function isRepeatedBlockerProposal(
return (
proposal.status === 'blocked' &&
proposal.blockerKind !== 'authority' &&
proposal.blockerKind !== 'external'
proposal.blockerKind !== 'external' &&
proposal.blockerKind !== 'infeasible'
);
}

/**
* Appended to `lastReason` when an `infeasible` blocker is accepted, so the
* stopped Goal tells the user what to do rather than only what went wrong.
* The verifier's reason says why the objective cannot hold; this says that
* resuming as-is will not change that.
*/
export const GOAL_INFEASIBLE_NEXT_STEP =
'The objective cannot be satisfied as written; edit or replace the Goal with an objective the evidence allows before resuming it.';

export function validateGoalProposalReason(reason: string): string | null {
if (!reason.trim()) return 'Goal proposal reason must not be empty';
if ([...reason].length > GOAL_PROPOSAL_REASON_MAX_CHARACTERS) {
Expand Down
77 changes: 77 additions & 0 deletions packages/core/src/goals/goal-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { describe, expect, it, vi } from 'vitest';
import type { GoalEvidenceRecord } from './goal-evidence.js';
import type { GoalRecoveryRecord } from './goal-persistence.js';
import {
GOAL_INFEASIBLE_NEXT_STEP,
GOAL_CHECKPOINT_CLAIM_LIMIT,
GOAL_CHECKPOINT_REQUEST_TOO_LARGE_REASON,
GOAL_CHECKPOINT_STALL_LIMIT,
Expand Down Expand Up @@ -627,6 +628,82 @@ describe('goal runtime', () => {
);
});

it('accepts an evidenced infeasible blocker on its first turn, with the next step spelled out', async () => {
const journal = fakeGoalJournal();
let records: readonly RuntimeRecord[] = [];
const evidenceSource = fakeEvidenceSource(() => records);
const verifier: GoalVerifier = vi.fn(async () => ({
decision: 'accept' as const,
reason: 'The named branch does not exist',
}));
const host = fakeGoalTurnHost();
const runtime = createGoalRuntime({ journal, evidenceSource, verifier });
runtime.bindHost(host);
await runtime.dispatch({
action: 'create',
objective: 'Rebase onto the v9 branch',
});
const permit = host.started[0]!;
const cursorId = runtime.getSnapshot().goal!.evidenceCursor.recordId!;
const base = verifierEvidenceRecords(permit, cursorId, 'probe');
records = [
base[0]!,
{
...base[1]!,
type: 'tool_result',
provenance: 'tool_result',
message: {
role: 'user',
parts: [
{
functionResponse: {
name: 'shell',
response: { output: "fatal: branch 'v9' not found" },
},
},
],
},
},
];

// No three-turn streak: the whole point is to stop before the budget
// does, and the evidence bar (an external fact) is what earns that.
const receipt = runtime.recordTerminalProposal(permit, {
status: 'blocked',
blockerKind: 'infeasible',
reason:
'Checked the remote: no v9 branch exists, so nothing in scope can rebase onto it.',
evidenceRefs: ['probe'],
});
expect(receipt).toEqual({ recorded: true, readyForVerification: true });

await runtime.finishTurn(permit);

expect(verifier).toHaveBeenCalledWith(
expect.objectContaining({
proposal: expect.objectContaining({ blockerKind: 'infeasible' }),
blockedPolicy: expect.stringContaining(
'An infeasible blocker may also be accepted immediately',
),
}),
expect.any(AbortSignal),
);
expect(journal.appended.map((payload) => payload.cause)).toEqual([
'create',
'turn_finished',
'verifier_accept',
'blocked',
]);
expect(runtime.getSnapshot()).toMatchObject({
activity: 'idle',
goal: {
status: 'blocked',
lastReason: `The named branch does not exist ${GOAL_INFEASIBLE_NEXT_STEP}`,
},
});
expect(host.started).toHaveLength(1);
});

it('rejects an invalid evidence reference without calling the verifier', async () => {
const journal = fakeGoalJournal();
let records: readonly RuntimeRecord[] = [];
Expand Down
8 changes: 6 additions & 2 deletions packages/core/src/goals/goal-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
GOAL_CHECKPOINT_STALLED_REASON,
GOAL_DEFAULT_TOKEN_BUDGET,
GOAL_EVIDENCE_CATALOG_EXHAUSTED_REASON,
GOAL_INFEASIBLE_NEXT_STEP,
GOAL_STATE_VERSION,
goalTokenBudgetReason,
isGoalTokenBudgetSpent,
Expand Down Expand Up @@ -614,7 +615,7 @@ export function createGoalRuntime(
...base,
proposal: { ...attempt.proposal, status: 'blocked' },
blockedPolicy:
'A blocked Goal is resumable. It may be accepted immediately only when the evidence shows that new user authority or a material user choice is required, or that an external state change is required, and no meaningful in-scope work remains. An ordinary technical blocker requires evidence of the same cause from the current and two immediately preceding Goal turns. Difficulty, uncertainty, incomplete work, or a preference for clarification do not by themselves justify blocked.',
'A blocked Goal is resumable. It may be accepted immediately only when the evidence shows that new user authority or a material user choice is required, or that an external state change is required, and no meaningful in-scope work remains. An infeasible blocker may also be accepted immediately, only when cited external_fact evidence shows the objective cannot be satisfied as written: it contradicts itself, it names a target that verifiably does not exist, or it requires an action outside what the tools can perform; reject it when the obstacle is difficulty, uncertainty, information the model could still obtain, or a preference to ask. An ordinary technical blocker requires evidence of the same cause from the current and two immediately preceding Goal turns. Difficulty, uncertainty, incomplete work, or a preference for clarification do not by themselves justify blocked.',
};
};

Expand Down Expand Up @@ -665,7 +666,10 @@ export function createGoalRuntime(
...snapshot.goal,
activeTimeMs: elapsedActiveTime(snapshot.goal, now),
updatedAt: now,
lastReason: outcome.result.reason,
lastReason:
attempt.proposal.blockerKind === 'infeasible'
? `${outcome.result.reason} ${GOAL_INFEASIBLE_NEXT_STEP}`
: outcome.result.reason,
};
const acceptedSnapshot: GoalSnapshotV2 = {
v: GOAL_STATE_VERSION,
Expand Down
10 changes: 10 additions & 0 deletions packages/core/src/goals/goal-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,16 @@ describe('UpdateGoalTool', () => {
expect(schema.properties.blockerKind.description).toContain(
'exact same reason text',
);
expect(
(schema.properties.blockerKind as { enum?: string[] }).enum,
).toContain('infeasible');
expect(schema.properties.blockerKind.description).toContain(
'cannot be satisfied as written',
);
expect(tool.description).toContain('a tool result, not your own text');
expect(tool.description).toContain(
'not for difficulty, uncertainty, information you could still obtain',
);
});

it('rejects lineage turn ids before recording a proposal', async () => {
Expand Down
9 changes: 5 additions & 4 deletions packages/core/src/goals/goal-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
} from './goal-runtime.js';
import { goalTurnContext } from './goal-turn-context.js';
import {
type GoalBlockerKind,
GOAL_PROPOSAL_REASON_MAX_CHARACTERS,
type GoalRecord,
type GoalSnapshotV2,
Expand Down Expand Up @@ -58,7 +59,7 @@ export interface UpdateGoalToolParams {
status: 'complete' | 'blocked';
reason: string;
evidenceRefs: string[];
blockerKind?: 'authority' | 'external' | 'repeated';
blockerKind?: GoalBlockerKind;
}

export type GoalToolResult = ToolResult;
Expand Down Expand Up @@ -370,7 +371,7 @@ export class UpdateGoalTool extends BaseDeclarativeTool<
super(
UpdateGoalTool.Name,
ToolDisplayNames.UPDATE_GOAL,
'Propose that the current Goal is complete or blocked. Before calling, call get_goal in the current turn and cite only values from evidenceCatalog.entries[].uuid, never goalId, turnId, or lineageTurnIds. If completion depends on user-facing content delivered in the current turn, emit only the content required by the objective, then call get_goal, wait for its result, and call update_goal in a later model step with the returned delivered_output UUID. Do not add progress or completion commentary when the objective requires an exact output format. For blocked proposals, use authority when a user or maintainer decision or permission is required, external when an unavailable external resource or capability is evidenced, and repeated for the same evidenced blocker with the exact same reason text across three consecutive Goal turns; omitting blockerKind follows the repeated-blocker audit. Core records at most one proposal for the exact permitted turn and queues eligible proposals for independent verification. This tool never changes the Goal lifecycle or claims a terminal result. Do not tell the user the Goal is complete or blocked. If this tool reports readyForVerification, end the turn without additional user-facing text; otherwise continue the turn without claiming a terminal result. The Goal status card reports the independent verification result.',
'Propose that the current Goal is complete or blocked. Before calling, call get_goal in the current turn and cite only values from evidenceCatalog.entries[].uuid, never goalId, turnId, or lineageTurnIds. If completion depends on user-facing content delivered in the current turn, emit only the content required by the objective, then call get_goal, wait for its result, and call update_goal in a later model step with the returned delivered_output UUID. Do not add progress or completion commentary when the objective requires an exact output format. For blocked proposals, use authority when a user or maintainer decision or permission is required, external when an unavailable external resource or capability is evidenced, repeated for the same evidenced blocker with the exact same reason text across three consecutive Goal turns, and infeasible when a cited external_fact (a tool result, not your own text) shows the objective cannot be satisfied as written -- it contradicts itself, names a target that verifiably does not exist, or needs an action no tool can perform; infeasible is not for difficulty, uncertainty, information you could still obtain, or wanting to ask, and its reason must state what was checked and why no in-scope work could satisfy the objective. Omitting blockerKind follows the repeated-blocker audit. Core records at most one proposal for the exact permitted turn and queues eligible proposals for independent verification. This tool never changes the Goal lifecycle or claims a terminal result. Do not tell the user the Goal is complete or blocked. If this tool reports readyForVerification, end the turn without additional user-facing text; otherwise continue the turn without claiming a terminal result. The Goal status card reports the independent verification result.',
Kind.Think,
{
type: 'object',
Expand All @@ -397,9 +398,9 @@ export class UpdateGoalTool extends BaseDeclarativeTool<
},
blockerKind: {
type: 'string',
enum: ['authority', 'external', 'repeated'],
enum: ['authority', 'external', 'repeated', 'infeasible'],
description:
'authority: a user or maintainer decision or permission is required; external: an evidenced external resource or capability is unavailable; repeated: the same evidenced blocker with the exact same reason text across three consecutive Goal turns. Omission uses the repeated-blocker audit.',
'authority: a user or maintainer decision or permission is required; external: an evidenced external resource or capability is unavailable; repeated: the same evidenced blocker with the exact same reason text across three consecutive Goal turns; infeasible: a cited external_fact shows the objective cannot be satisfied as written (self-contradictory, names a target that verifiably does not exist, or needs an action no tool can perform) -- not difficulty, uncertainty, or obtainable information. Omission uses the repeated-blocker audit.',
},
},
required: ['status', 'reason', 'evidenceRefs'],
Expand Down
Loading