Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 166 additions & 10 deletions packages/core/src/goals/goal-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,23 @@ describe('GetGoalTool', () => {
});

describe('UpdateGoalTool', () => {
const activeSnapshot = () => ({
v: 2 as const,
activity: 'running' as const,
goal: {
goalId: permit.goalId,
revision: permit.revision,
objective: 'Deliver the result',
status: 'active' as const,
evidenceCursor: { recordId: 'goal-created' },
turnCount: 3,
activeTimeMs: 100,
tokensUsed: 0,
createdAt: 1,
updatedAt: 2,
},
});

it('exposes the exact evidence and non-terminal response contract', () => {
const tool = new UpdateGoalTool(makeConfig({}));
const schema = tool.schema.parametersJsonSchema as {
Expand Down Expand Up @@ -493,8 +510,11 @@ describe('UpdateGoalTool', () => {
expect(recordTerminalProposal).not.toHaveBeenCalled();
});

it('rejects completion that omits current delivered output', async () => {
const recordTerminalProposal = vi.fn();
it("cites this turn's delivered output for a completion that omitted it", async () => {
const recordTerminalProposal = vi.fn(() => ({
recorded: true,
readyForVerification: true,
}));
const getGoalForWorker = vi.fn().mockResolvedValue({
goalId: permit.goalId,
revision: permit.revision,
Expand Down Expand Up @@ -553,15 +573,151 @@ describe('UpdateGoalTool', () => {

const result = await invocation.execute(new AbortController().signal);

expect(JSON.parse(String(result.llmContent))).toEqual({
proposalRecorded: false,
readyForVerification: false,
goalLifecycleChanged: false,
uncitedCurrentDeliveredOutput: ['letter-x'],
error:
'The completion proposal omitted delivered output from the current Goal turn. Call get_goal after delivering the final output, then retry update_goal with the returned evidenceCatalog UUIDs.',
// Refusing here could not converge: complying emits assistant text, which
// is delivered_output stamped with this same turn, so the required set
// grew by one per retry until a human stopped the Goal.
expect(recordTerminalProposal).toHaveBeenCalledWith(
permit,
expect.objectContaining({
status: 'complete',
evidenceRefs: ['tool-result-1', 'letter-x'],
}),
);
expect(JSON.parse(String(result.llmContent))).toMatchObject({
proposalRecorded: true,
readyForVerification: true,
autoCitedCurrentDeliveredOutput: ['letter-x'],
});
expect(recordTerminalProposal).not.toHaveBeenCalled();
});

it('does not duplicate output the completion already cited', async () => {
// validateGoalEvidenceReferences rejects a duplicated ref outright, so the
// fold has to be a union rather than an append.
const recordTerminalProposal = vi.fn(() => ({
recorded: true,
readyForVerification: true,
}));
const tool = new UpdateGoalTool(
makeConfig({
getGoalForWorker: vi.fn().mockResolvedValue({
goalId: permit.goalId,
revision: permit.revision,
objective: 'Deliver the result',
evidenceCursor: { recordId: 'goal-created' },
evidenceCatalog: {
entries: [
{
uuid: 'letter-x',
provenance: 'assistant_output',
turnId: permit.turnId,
preview: 'X',
proofKind: 'delivered_output',
},
{
uuid: 'letter-y',
provenance: 'assistant_output',
turnId: permit.turnId,
preview: 'Y',
proofKind: 'delivered_output',
},
{
uuid: 'current-external-fact',
provenance: 'tool_result',
turnId: permit.turnId,
preview: 'permission denied',
proofKind: 'external_fact',
},
{
uuid: 'prior-delivered-output',
Comment thread
qwen-code-dev-bot marked this conversation as resolved.
provenance: 'assistant_output',
turnId: 'prior-turn',
preview: 'Earlier output',
proofKind: 'delivered_output',
},
],
lineageTurnIds: ['prior-turn', permit.turnId],
},
}),
getSnapshotForPermit: vi.fn(() => activeSnapshot()),
recordTerminalProposal,
}),
);
const invocation = goalTurnContext.run(permit, () =>
tool.build({
status: 'complete',
reason: 'Delivered',
evidenceRefs: ['letter-x'],
}),
);

const result = await invocation.execute(new AbortController().signal);

expect(recordTerminalProposal).toHaveBeenCalledWith(
permit,
expect.objectContaining({ evidenceRefs: ['letter-x', 'letter-y'] }),
);
expect(recordTerminalProposal).toHaveBeenCalledTimes(1);
Comment thread
qwen-code-dev-bot marked this conversation as resolved.
expect(JSON.parse(String(result.llmContent))).toMatchObject({
autoCitedCurrentDeliveredOutput: ['letter-y'],
});
Comment thread
qwen-code-dev-bot marked this conversation as resolved.
});

it('leaves a blocked proposal to cite whatever it chose', async () => {
// The gate only ever guarded completion: a blocker is judged on the
// blocker, not on what the turn happened to deliver.
const recordTerminalProposal = vi.fn(() => ({
recorded: true,
readyForVerification: false,
}));
const tool = new UpdateGoalTool(
makeConfig({
getGoalForWorker: vi.fn().mockResolvedValue({
goalId: permit.goalId,
revision: permit.revision,
objective: 'Deliver the result',
evidenceCursor: { recordId: 'goal-created' },
evidenceCatalog: {
entries: [
{
uuid: 'tool-result-1',
provenance: 'tool_result',
turnId: permit.turnId,
preview: 'permission denied',
proofKind: 'external_fact',
},
{
uuid: 'letter-x',
provenance: 'assistant_output',
turnId: permit.turnId,
preview: 'X',
proofKind: 'delivered_output',
},
],
lineageTurnIds: [permit.turnId],
},
}),
getSnapshotForPermit: vi.fn(() => activeSnapshot()),
recordTerminalProposal,
}),
);
const invocation = goalTurnContext.run(permit, () =>
tool.build({
status: 'blocked',
reason: 'The credential store is unreadable',
evidenceRefs: ['tool-result-1'],
blockerKind: 'external',
}),
);

const result = await invocation.execute(new AbortController().signal);

expect(recordTerminalProposal).toHaveBeenCalledWith(
permit,
expect.objectContaining({ evidenceRefs: ['tool-result-1'] }),
);
expect(JSON.parse(String(result.llmContent))).not.toHaveProperty(
'autoCitedCurrentDeliveredOutput',
);
});

it('queues truncated completion for boundary classification', async () => {
Expand Down
63 changes: 35 additions & 28 deletions packages/core/src/goals/goal-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ class UpdateGoalInvocation extends BaseToolInvocation<
) {
throw staleGoalTurnError();
}
let autoCitedCurrentDeliveredOutput: string[] = [];
const evidenceEntries = view.evidenceCatalog?.entries;
if (evidenceEntries) {
const normalizedEvidenceRefs = this.params.evidenceRefs.map((reference) =>
Expand Down Expand Up @@ -242,38 +243,38 @@ class UpdateGoalInvocation extends BaseToolInvocation<
};
}
const citedEvidenceRefs = new Set(normalizedEvidenceRefs);
const uncitedCurrentDeliveredOutput = evidenceEntries
.filter(
(entry) =>
entry.proofKind === 'delivered_output' &&
entry.turnId === permit.turnId &&
!citedEvidenceRefs.has(entry.uuid),
)
.map((entry) => entry.uuid);
if (
this.params.status === 'complete' &&
uncitedCurrentDeliveredOutput.length > 0
) {
return {
llmContent: JSON.stringify({
proposalRecorded: false,
readyForVerification: false,
goalLifecycleChanged: false,
uncitedCurrentDeliveredOutput,
error:
'The completion proposal omitted delivered output from the current Goal turn. Call get_goal after delivering the final output, then retry update_goal with the returned evidenceCatalog UUIDs.',
}),
returnDisplay:
'Goal proposal was not recorded because the current delivered output was not cited. Read the current Goal and retry.',
};
}
// The verifier judges a completion against this turn's delivered output,
// so it has to be cited. Asking the model to cite it cannot converge:
// assistant output is `delivered_output` stamped with this same turn, so
// every attempt to comply — reading the catalog, then calling back —
// emits text that becomes another uncited entry, and the required set
// grows by one per retry. Refusing produced runs that proposed
// completion until a human paused them.
//
// Nothing about the list needs the model's judgment: it is exactly the
// entries computed here. Fold them in instead of demanding they be
// repeated back. Both sets are drawn from the same catalog and are
// disjoint by construction, so the union cannot exceed
// GOAL_EVIDENCE_REFERENCE_LIMIT, which is that catalog's own entry cap.
autoCitedCurrentDeliveredOutput =
this.params.status === 'complete'
Comment thread
qwen-code-dev-bot marked this conversation as resolved.
? evidenceEntries
.filter(
(entry) =>
entry.proofKind === 'delivered_output' &&
entry.turnId === permit.turnId &&
Comment thread
qwen-code-dev-bot marked this conversation as resolved.
!citedEvidenceRefs.has(entry.uuid),
)
.map((entry) => entry.uuid)
: [];
}
const proposal: GoalTerminalProposal = {
status: this.params.status,
reason: this.params.reason.trim(),
evidenceRefs: this.params.evidenceRefs.map((reference) =>
reference.trim(),
),
evidenceRefs: [
...this.params.evidenceRefs.map((reference) => reference.trim()),
...autoCitedCurrentDeliveredOutput,
],
...(this.params.blockerKind
? { blockerKind: this.params.blockerKind }
: {}),
Expand All @@ -289,6 +290,12 @@ class UpdateGoalInvocation extends BaseToolInvocation<
proposalRecorded: receipt.recorded,
readyForVerification: receipt.readyForVerification,
goalLifecycleChanged: false,
// Reported so the proposal the verifier sees is not a surprise, and so a
// model that wants to cite this turn's output explicitly can see it was
// already covered rather than calling back to add it.
...(autoCitedCurrentDeliveredOutput.length > 0
? { autoCitedCurrentDeliveredOutput }
: {}),
nextAction: receipt.readyForVerification
? 'End this turn without user-facing text. Do not claim the Goal is complete or blocked. The Goal status card will report the independent verification result.'
: 'Continue this turn without claiming the Goal is complete or blocked. A repeated-blocker audit requires the same blocker mode and exact same reason text across three consecutive Goal turns, with current evidence cited on each turn.',
Expand Down
Loading