From 6ce0598ae8f5f09aec8008b6b4d1811cb2b2ecfc Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 10:33:20 +0800 Subject: [PATCH 01/10] fix(headless): accept canonical trajectory refs --- .../fixtures/runtime-event-validation-corpus.json | 3 +++ packages/headless/harbor/maka_trajectory.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/packages/core/src/__tests__/fixtures/runtime-event-validation-corpus.json b/packages/core/src/__tests__/fixtures/runtime-event-validation-corpus.json index e6d4acf59b..7e82dc0aef 100644 --- a/packages/core/src/__tests__/fixtures/runtime-event-validation-corpus.json +++ b/packages/core/src/__tests__/fixtures/runtime-event-validation-corpus.json @@ -78,9 +78,12 @@ "traceEventId": "trace-1", "toolCallId": "tool-call-1", "providerEventId": "provider-event-1", + "sourceMessageDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "providerRequestTraceId": "provider-trace-1", "artifactId": "artifact-1", "operationId": "operation-1", + "parentToolCallId": "parent-tool-call-1", + "parentOperationId": "parent-operation-1", "stepId": "step-1", "sourceInvocationId": "source-invocation-1", "sourceRunId": "source-run-1", diff --git a/packages/headless/harbor/maka_trajectory.py b/packages/headless/harbor/maka_trajectory.py index 5e1d9c3de9..bc76e7e773 100644 --- a/packages/headless/harbor/maka_trajectory.py +++ b/packages/headless/harbor/maka_trajectory.py @@ -163,9 +163,12 @@ class _IncompleteTrajectoryEvidence(ValueError): "traceEventId", "toolCallId", "providerEventId", + "sourceMessageDigest", "providerRequestTraceId", "artifactId", "operationId", + "parentToolCallId", + "parentOperationId", "stepId", "sourceInvocationId", "sourceRunId", @@ -1786,6 +1789,10 @@ def _is_runtime_refs(refs: Any) -> bool: string_refs = _RUNTIME_EVENT_REF_KEYS - {"sourceRuntimeEventHighWater"} if any(key in refs and not isinstance(refs[key], str) for key in string_refs): return False + if "sourceMessageDigest" in refs and re.fullmatch( + r"sha256:[0-9a-f]{64}", refs["sourceMessageDigest"] + ) is None: + return False if "sourceRuntimeEventHighWater" not in refs: return True return _is_nonnegative_safe_integer(refs["sourceRuntimeEventHighWater"]) From 8c8f6565a6b0ecade87f157b53520c9e01a1580a Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 10:40:58 +0800 Subject: [PATCH 02/10] fix(headless): preserve token usage provenance --- .../src/__tests__/cell-output.test.ts | 2 + .../__tests__/fixed-prompt-controller.test.ts | 15 +++++-- .../src/__tests__/harbor-cell.test.ts | 6 +-- .../src/__tests__/harbor-task-runner.test.ts | 3 ++ .../src/__tests__/harness-ab-report.test.ts | 40 +++++++++++++++++++ packages/headless/src/ab-summary.ts | 5 ++- packages/headless/src/cell-output.ts | 15 ++++++- .../headless/src/fixed-prompt-controller.ts | 6 ++- .../headless/src/fixed-prompt-wal-types.ts | 1 + packages/headless/src/harbor-cell.ts | 2 +- packages/headless/src/harbor-task-runner.ts | 5 ++- 11 files changed, 86 insertions(+), 14 deletions(-) diff --git a/packages/headless/src/__tests__/cell-output.test.ts b/packages/headless/src/__tests__/cell-output.test.ts index 35535db220..b266294410 100644 --- a/packages/headless/src/__tests__/cell-output.test.ts +++ b/packages/headless/src/__tests__/cell-output.test.ts @@ -171,6 +171,7 @@ describe('Harbor cell output contract', () => { costUsd: 0.00523, pricingSource: 'runtime', }, + tokenSummarySource: 'final', toolSummary: { providerVisibleToolCount: 6, actualToolCalls: 2, @@ -407,6 +408,7 @@ describe('Harbor cell output contract', () => { }); assert.equal(output.steps, 2); + assert.equal(output.tokenSummarySource, 'checkpoint'); }); test('summarizes context budget diagnostics from token usage events', () => { diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 8694101d62..09821ae827 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -46,12 +46,15 @@ describe('fixed prompt controller', () => { systemPromptPath, resultsJsonlPath: join(dir, 'results.jsonl'), tasks: [{ id: 'task-a', path: '/bench/task-a' }], - taskRunner: async () => harborOutput({ taskId: 'task-a', verifier }), + taskRunner: async () => + harborOutput({ taskId: 'task-a', verifier, tokenSummarySource: 'final' }), }); - assert.equal(result.events[0]?.type, 'task_completed'); - if (result.events[0]?.type === 'task_completed') - assert.deepEqual(result.events[0].harbor.verifier, verifier); + const event = result.events[0]; + assert.equal(event?.type, 'task_completed'); + if (event?.type !== 'task_completed') assert.fail('expected completed event'); + assert.deepEqual(event.harbor.verifier, verifier); + assert.equal(event.tokenSummarySource, 'final'); }); }); @@ -1300,6 +1303,7 @@ describe('fixed prompt controller', () => { taskId: 'task-a', status: 'failed', errorClass: 'auth', + tokenSummarySource: 'checkpoint', executionIdentity: { llmConnectionSlug: 'fake', model: 'fake-model', @@ -1331,6 +1335,7 @@ describe('fixed prompt controller', () => { if (event?.type !== 'task_budget_exhausted') assert.fail('expected budget exhaustion event'); assert.equal(event.eligible, false); assert.equal(event.evidenceErrorClass, 'auth'); + assert.equal(event.tokenSummarySource, 'checkpoint'); assert.equal(result.stopReason, 'systemic_provider_failure'); }); }); @@ -3593,6 +3598,7 @@ function harborOutput(input: { promptHash?: string; omitPromptHash?: boolean; tokenSummary?: TaskRunOutput['cell']['tokenSummary']; + tokenSummarySource?: 'final' | 'checkpoint'; omitTokenSummary?: boolean; contextBudgetPolicy?: TaskRunOutput['cell']['contextBudgetPolicy']; contextBudgetSummary?: TaskRunOutput['cell']['contextBudgetSummary']; @@ -3636,6 +3642,7 @@ function harborOutput(input: { tokenSummary: input.tokenSummary ?? tokenSummary({ input: 1, output: 2, reasoning: 0, total: 3, costUsd: 0.02 }), + ...(input.tokenSummarySource ? { tokenSummarySource: input.tokenSummarySource } : {}), }), ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}), ...(input.contextBudgetSummary ? { contextBudgetSummary: input.contextBudgetSummary } : {}), diff --git a/packages/headless/src/__tests__/harbor-cell.test.ts b/packages/headless/src/__tests__/harbor-cell.test.ts index bf742c0863..c3e4ad65fc 100644 --- a/packages/headless/src/__tests__/harbor-cell.test.ts +++ b/packages/headless/src/__tests__/harbor-cell.test.ts @@ -901,10 +901,8 @@ describe('runHarborCell', () => { costUsd: 0.012, pricingSource: 'runtime', }); - assert.deepEqual( - JSON.parse(await readFile(result.outputPath, 'utf8')).tokenSummary, - result.output.tokenSummary, - ); + assert.equal(result.output.tokenSummarySource, 'checkpoint'); + assert.deepEqual(JSON.parse(await readFile(result.outputPath, 'utf8')), result.output); }); }); diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 440bcc20f4..a6af928be5 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -1212,6 +1212,7 @@ describe('createHarborTaskRunner', () => { costUsd: 0, pricingSource: 'runtime', }); + assert.equal(output.cell.tokenSummarySource, 'final'); } finally { await new Promise((resolve, reject) => upstream.close((error) => (error ? reject(error) : resolve())), @@ -1880,6 +1881,7 @@ describe('createHarborTaskRunner', () => { const output = await runner(runInput()); assert.deepEqual(output.cell.tokenSummary, usageCheckpoint); + assert.equal(output.cell.tokenSummarySource, 'checkpoint'); }); }); @@ -2335,6 +2337,7 @@ describe('createHarborTaskRunner', () => { assert.ok(error instanceof FixedPromptBudgetExhaustedError); assert.deepEqual(error.artifactRefs?.tokenSummary, usageCheckpoint); assert.deepEqual(error.artifactRefs?.cellOutput?.tokenSummary, usageCheckpoint); + assert.equal(error.artifactRefs?.cellOutput?.tokenSummarySource, 'checkpoint'); return true; }); }); diff --git a/packages/headless/src/__tests__/harness-ab-report.test.ts b/packages/headless/src/__tests__/harness-ab-report.test.ts index 6bbf57d8c0..8b0df3e250 100644 --- a/packages/headless/src/__tests__/harness-ab-report.test.ts +++ b/packages/headless/src/__tests__/harness-ab-report.test.ts @@ -460,6 +460,46 @@ describe('harness A/B report', () => { assert.equal(report.coverage.missingFinalUsageCells, 1); }); + test('keeps completed-cell checkpoints out of final metering', () => { + const checkpointOnlyCompletion = { + ...usage('a', true, 100, 40, 20, 0.1), + tokenSummarySource: 'checkpoint' as const, + }; + const summary = summarizeAbComparison({ + runId: 'glm-harness-ab', + roundId: 'ab-summary', + baselineArmId: 'maka', + candidateArmId: 'opencode', + evaluationTaskIds: ['a'], + baselineRuns: [[checkpointOnlyCompletion]], + candidateRuns: [[usage('a', true, 120, 50, 30, 0.2)]], + }); + + assert.equal(summary.pairedAttempts.fullyMeteredPairs, 0); + assert.deepEqual(summary.pairedAttempts.missingUsagePairIds, ['a#r0']); + assert.equal(summary.baseline.missingFinalUsage, 1); + }); + + test('keeps failed legacy cell usage out of final metering', () => { + const legacyFailure = { + ...usage('a', true, 100, 40, 20, 0.1), + status: 'failed' as const, + }; + const summary = summarizeAbComparison({ + runId: 'glm-harness-ab', + roundId: 'ab-summary', + baselineArmId: 'maka', + candidateArmId: 'opencode', + evaluationTaskIds: ['a'], + baselineRuns: [[legacyFailure]], + candidateRuns: [[usage('a', true, 120, 50, 30, 0.2)]], + }); + + assert.equal(summary.pairedAttempts.fullyMeteredPairs, 0); + assert.deepEqual(summary.pairedAttempts.missingUsagePairIds, ['a#r0']); + assert.equal(summary.baseline.missingFinalUsage, 1); + }); + test('preserves an early stop and rejects completion', () => { const summary = summarizeAbComparison({ runId: 'glm-harness-ab', diff --git a/packages/headless/src/ab-summary.ts b/packages/headless/src/ab-summary.ts index 358896ec0c..115ac5b621 100644 --- a/packages/headless/src/ab-summary.ts +++ b/packages/headless/src/ab-summary.ts @@ -672,7 +672,10 @@ function hasCompleteTokenSummary( event: FixedPromptTaskWalEvent, ): event is FixedPromptTaskWalEvent & { tokenSummary: HarborCellTokenSummary } { if (!hasTokenSummary(event)) return false; - return event.type !== 'task_budget_exhausted' || event.tokenSummarySource === 'final'; + if (event.type === 'task_budget_exhausted') return event.tokenSummarySource === 'final'; + if (event.type !== 'task_completed') return false; + if (event.tokenSummarySource !== undefined) return event.tokenSummarySource === 'final'; + return event.status === 'completed'; } function hasTokenSummary( diff --git a/packages/headless/src/cell-output.ts b/packages/headless/src/cell-output.ts index 4f5fe05702..723b8f68b7 100644 --- a/packages/headless/src/cell-output.ts +++ b/packages/headless/src/cell-output.ts @@ -202,6 +202,7 @@ export interface HarborCellOutput { executionIdentity?: HarborCellExecutionIdentity; deadlineSettlement?: HarborCellDeadlineSettlement; tokenSummary?: HarborCellTokenSummary; + tokenSummarySource?: 'final' | 'checkpoint'; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; continuationSummary?: HarborCellContinuationSummary; @@ -235,7 +236,13 @@ export function buildHarborCellOutput(input: { ...(promptHash ? { promptHash } : {}), ...(input.executionIdentity ? { executionIdentity: input.executionIdentity } : {}), ...(input.deadlineSettlement ? { deadlineSettlement: input.deadlineSettlement } : {}), - ...(tokenSummary ? { tokenSummary } : {}), + ...(tokenSummary + ? { + tokenSummary, + tokenSummarySource: + invocation.status === 'completed' ? ('final' as const) : ('checkpoint' as const), + } + : {}), ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}), ...contextBudgetSummaryField(invocation.events), ...(input.continuationSummary ? { continuationSummary: input.continuationSummary } : {}), @@ -323,6 +330,11 @@ export function validateHarborCellOutput(value: unknown): HarborCellOutput { : undefined; const tokenSummary = 'tokenSummary' in value ? validateHarborCellTokenSummary(value.tokenSummary) : undefined; + const tokenSummarySource = requireOptionalStringUnion( + value.tokenSummarySource, + 'tokenSummarySource', + ['final', 'checkpoint'] as const, + ); const contextBudgetPolicy = 'contextBudgetPolicy' in value ? validateContextBudgetPolicySnapshot(value.contextBudgetPolicy) @@ -352,6 +364,7 @@ export function validateHarborCellOutput(value: unknown): HarborCellOutput { ...(executionIdentity !== undefined ? { executionIdentity } : {}), ...(deadlineSettlement !== undefined ? { deadlineSettlement } : {}), ...(tokenSummary ? { tokenSummary } : {}), + ...(tokenSummarySource ? { tokenSummarySource } : {}), ...(contextBudgetPolicy !== undefined ? { contextBudgetPolicy } : {}), ...(contextBudgetSummary !== undefined ? { contextBudgetSummary } : {}), ...(continuationSummary !== undefined ? { continuationSummary } : {}), diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 39b38bedff..d4bedeb7a2 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -732,6 +732,9 @@ function taskCompletedEvent(input: { ? { deadlineSettlement: output.cell.deadlineSettlement } : {}), ...(output.cell.tokenSummary ? { tokenSummary: output.cell.tokenSummary } : {}), + ...(output.cell.tokenSummarySource + ? { tokenSummarySource: output.cell.tokenSummarySource } + : {}), ...(output.cell.contextBudgetPolicy ? { contextBudgetPolicy: output.cell.contextBudgetPolicy } : {}), @@ -1093,7 +1096,8 @@ function taskBudgetExhaustedEvent(input: { const tokenSummary = artifactRefs.cellOutput?.tokenSummary ?? artifactRefs.tokenSummary; const tokenSummarySource = tokenSummary ? artifactRefs.cellOutput - ? 'final' + ? (artifactRefs.cellOutput.tokenSummarySource ?? + (artifactRefs.cellOutput.status === 'completed' ? 'final' : 'checkpoint')) : 'checkpoint' : undefined; const executionIdentity = diff --git a/packages/headless/src/fixed-prompt-wal-types.ts b/packages/headless/src/fixed-prompt-wal-types.ts index acf2c900b7..6683617baa 100644 --- a/packages/headless/src/fixed-prompt-wal-types.ts +++ b/packages/headless/src/fixed-prompt-wal-types.ts @@ -55,6 +55,7 @@ export interface FixedPromptTaskCompletedEvent { runtimeRefs?: HarborCellRuntimeRefs; deadlineSettlement?: HarborCellDeadlineSettlement; tokenSummary?: HarborCellTokenSummary; + tokenSummarySource?: 'final' | 'checkpoint'; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; continuationSummary?: HarborCellContinuationSummary; diff --git a/packages/headless/src/harbor-cell.ts b/packages/headless/src/harbor-cell.ts index b7e170a09d..65a1235c42 100644 --- a/packages/headless/src/harbor-cell.ts +++ b/packages/headless/src/harbor-cell.ts @@ -671,7 +671,7 @@ export async function writeHarborCellArtifacts( const tokenSummary = selectHarborCellTokenSummary(rawOutput.tokenSummary, usageCheckpoint); const output = tokenSummary && tokenSummary !== rawOutput.tokenSummary - ? { ...rawOutput, tokenSummary } + ? { ...rawOutput, tokenSummary, tokenSummarySource: 'checkpoint' as const } : rawOutput; await writeHarborCellArtifact(outputPath, `${JSON.stringify(output, null, 2)}\n`); return { output, outputPath, runtimeEventsPath }; diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 6d054cd93e..10f3d00446 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -616,7 +616,7 @@ export function createHarborTaskRunner(options: HarborTaskRunnerOptions): TaskRu const selectedUsage = selectHarborCellTokenSummary(rawCell.tokenSummary, usageCheckpoint); const checkpointedCell = selectedUsage && selectedUsage !== rawCell.tokenSummary - ? { ...rawCell, tokenSummary: selectedUsage } + ? { ...rawCell, tokenSummary: selectedUsage, tokenSummarySource: 'checkpoint' as const } : rawCell; const usageCell = checkpointedCell.tokenSummary || !providerUsage || !runnerOptions.pricing @@ -624,6 +624,7 @@ export function createHarborTaskRunner(options: HarborTaskRunnerOptions): TaskRu : { ...checkpointedCell, tokenSummary: providerTokenSummary(providerUsage, runnerOptions.pricing), + tokenSummarySource: 'final' as const, }; const cell = completeTimedOutTrial ? { @@ -944,7 +945,7 @@ export async function readTimedOutTrialArtifacts( const selectedUsage = selectHarborCellTokenSummary(cell.tokenSummary, usageCheckpoint); const recoveredCell = selectedUsage && selectedUsage !== cell.tokenSummary - ? { ...cell, tokenSummary: selectedUsage } + ? { ...cell, tokenSummary: selectedUsage, tokenSummarySource: 'checkpoint' as const } : cell; return cellArtifactRefs( recoveredCell, From 7bdc22b3815b340c7cbec452311304297d752a69 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 10:41:49 +0800 Subject: [PATCH 03/10] fix(headless): timestamp settled benchmark events --- .../src/__tests__/fixed-prompt-controller.test.ts | 9 +++++++-- packages/headless/src/fixed-prompt-controller.ts | 12 +++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 09821ae827..21eeffdc63 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -39,6 +39,7 @@ describe('fixed prompt controller', () => { outcome: 'passed' as const, attempts: [{ attempt: 1, classification: 'passed' as const, durationMs: 12, reward: 1 }], }; + let clock = 100; const result = await runFixedPromptController({ runId: 'run-1', roundId: 'round-1', @@ -46,8 +47,11 @@ describe('fixed prompt controller', () => { systemPromptPath, resultsJsonlPath: join(dir, 'results.jsonl'), tasks: [{ id: 'task-a', path: '/bench/task-a' }], - taskRunner: async () => - harborOutput({ taskId: 'task-a', verifier, tokenSummarySource: 'final' }), + taskRunner: async () => { + clock = 250; + return harborOutput({ taskId: 'task-a', verifier, tokenSummarySource: 'final' }); + }, + now: () => clock, }); const event = result.events[0]; @@ -55,6 +59,7 @@ describe('fixed prompt controller', () => { if (event?.type !== 'task_completed') assert.fail('expected completed event'); assert.deepEqual(event.harbor.verifier, verifier); assert.equal(event.tokenSummarySource, 'final'); + assert.equal(event.ts, 250); }); }); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index d4bedeb7a2..aecd313fd3 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -302,7 +302,6 @@ export async function runFixedPromptController( billingMode: input.billingMode, resumeFingerprint: input.resumeFingerprint, id: newId(), - ts: now(), newId, now, }).then((event) => ({ index, event })), @@ -498,7 +497,6 @@ async function runTaskAndBuildEvent(input: { billingMode?: HarborBillingMode; resumeFingerprint?: string; id: string; - ts: number; newId: () => string; now: () => number; }): Promise { @@ -544,7 +542,7 @@ async function runTaskAndBuildEvent(input: { billingMode: input.billingMode, resumeFingerprint: input.resumeFingerprint, id: input.id, - ts: input.ts, + ts: input.now(), }); } if (input.input.protectPassAtOne || input.input.infraFailurePolicy === 'terminal') { @@ -555,7 +553,7 @@ async function runTaskAndBuildEvent(input: { roundId: input.input.roundId, resumeFingerprint: input.resumeFingerprint, id: input.id, - ts: input.ts, + ts: input.now(), }); } // #64: a thrown Harbor/Docker error is an infra failure, often a transient @@ -583,7 +581,7 @@ async function runTaskAndBuildEvent(input: { billingMode: input.billingMode, resumeFingerprint: input.resumeFingerprint, id: input.id, - ts: input.ts, + ts: input.now(), }); } return taskInfraFailedEvent({ @@ -593,7 +591,7 @@ async function runTaskAndBuildEvent(input: { roundId: input.input.roundId, resumeFingerprint: input.resumeFingerprint, id: input.id, - ts: input.ts, + ts: input.now(), }); } } @@ -610,7 +608,7 @@ async function runTaskAndBuildEvent(input: { runId: input.input.runId, roundId: input.input.roundId, id: input.id, - ts: input.ts, + ts: input.now(), }); } From 0e85a92590ded1fa823b1fb849c1c04470683484 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 11:09:46 +0800 Subject: [PATCH 04/10] fix(headless): require explicit final usage evidence --- .../__tests__/fixed-prompt-controller.test.ts | 70 +++++++++++++++++++ .../src/__tests__/harness-ab-report.test.ts | 10 ++- .../src/__tests__/helpers/ab-run-fixtures.ts | 1 + .../__tests__/helpers/ab-summary-fixtures.ts | 1 + packages/headless/src/ab-summary.ts | 8 +-- packages/headless/src/cell-output.ts | 7 ++ .../headless/src/fixed-prompt-controller.ts | 9 ++- .../headless/src/fixed-prompt-wal-types.ts | 1 + 8 files changed, 92 insertions(+), 15 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 21eeffdc63..b89160d8db 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -1208,6 +1208,7 @@ describe('fixed prompt controller', () => { const retainedContextBudgetSummary = contextBudgetSummary({ prunedToolResults: 2 }); const cell = harborOutput({ taskId: 'task-a', + tokenSummarySource: 'final', contextBudgetPolicy: { enabled: true, minRecentTurns: 2 }, contextBudgetSummary: retainedContextBudgetSummary, executionIdentity: { @@ -1257,6 +1258,35 @@ describe('fixed prompt controller', () => { }); }); + test('keeps legacy completed timeout usage provisional without provenance', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const cell = harborOutput({ taskId: 'task-a' }).cell; + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => { + throw new FixedPromptBudgetExhaustedError('agent timed out', undefined, { + cellOutput: cell, + }); + }, + now: () => 100, + newId: idFactory(), + }); + + const event = result.events[0]; + assert.equal(event?.type, 'task_budget_exhausted'); + if (event?.type !== 'task_budget_exhausted') assert.fail('expected budget exhaustion event'); + assert.equal(event.tokenSummarySource, 'checkpoint'); + }); + }); + test('keeps an early-attested timeout eligible without claiming complete usage', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); @@ -2293,6 +2323,7 @@ describe('fixed prompt controller', () => { reward: 0, status: 'failed', errorClass: 'aborted', + tokenSummarySource: 'final', deadlineSettlement: { source: 'benchmark.deadline', mode: 'immediate' }, verifier: { outcome: 'failed', @@ -2581,6 +2612,45 @@ describe('fixed prompt controller', () => { }); }); + test('rejects checkpoint usage when final usage is required', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + requireExecutionIdentity: true, + requireFinalUsage: true, + expectedPricingProfile: 'test-profile', + taskRunner: async () => + harborOutput({ + taskId: 'task-a', + tokenSummarySource: 'checkpoint', + executionIdentity: { + llmConnectionSlug: 'fake', + model: 'fake-model', + systemPromptHash: hashSystemPrompt('fixed prompt\n'), + pricingProfile: 'test-profile', + }, + }), + now: () => 100, + newId: idFactory(), + }); + + const event = result.events[0]; + assert.equal(event?.type, 'task_plumbing_failed'); + if (event?.type !== 'task_plumbing_failed') assert.fail('expected plumbing failure event'); + assert.equal(event.errorClass, 'missing_token_usage'); + assert.ok(event.tokenSummary); + assert.equal(event.tokenSummarySource, 'checkpoint'); + }); + }); + test('rejects a verifier-graded failed result when required final usage is missing', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/__tests__/harness-ab-report.test.ts b/packages/headless/src/__tests__/harness-ab-report.test.ts index 8b0df3e250..da2311e087 100644 --- a/packages/headless/src/__tests__/harness-ab-report.test.ts +++ b/packages/headless/src/__tests__/harness-ab-report.test.ts @@ -480,18 +480,16 @@ describe('harness A/B report', () => { assert.equal(summary.baseline.missingFinalUsage, 1); }); - test('keeps failed legacy cell usage out of final metering', () => { - const legacyFailure = { - ...usage('a', true, 100, 40, 20, 0.1), - status: 'failed' as const, - }; + test('keeps legacy usage without provenance out of final metering', () => { + const legacyUnknown = usage('a', true, 100, 40, 20, 0.1); + delete legacyUnknown.tokenSummarySource; const summary = summarizeAbComparison({ runId: 'glm-harness-ab', roundId: 'ab-summary', baselineArmId: 'maka', candidateArmId: 'opencode', evaluationTaskIds: ['a'], - baselineRuns: [[legacyFailure]], + baselineRuns: [[legacyUnknown]], candidateRuns: [[usage('a', true, 120, 50, 30, 0.2)]], }); diff --git a/packages/headless/src/__tests__/helpers/ab-run-fixtures.ts b/packages/headless/src/__tests__/helpers/ab-run-fixtures.ts index e3cd9aa988..696ff5b240 100644 --- a/packages/headless/src/__tests__/helpers/ab-run-fixtures.ts +++ b/packages/headless/src/__tests__/helpers/ab-run-fixtures.ts @@ -55,6 +55,7 @@ export function completed(taskId: string, passed: boolean): FixedPromptTaskCompl errorClass: passed ? undefined : 'verification_failed', promptHash: 'hash', tokenSummary: tokenSummary({ input: 1, output: 1, reasoning: 0, total: 2, costUsd: 0.01 }), + tokenSummarySource: 'final', steps: 1, durationMs: 100, runtimeEventsPath: `/logs/${taskId}/runtime-events.jsonl`, diff --git a/packages/headless/src/__tests__/helpers/ab-summary-fixtures.ts b/packages/headless/src/__tests__/helpers/ab-summary-fixtures.ts index f243286fef..d4f6cfadec 100644 --- a/packages/headless/src/__tests__/helpers/ab-summary-fixtures.ts +++ b/packages/headless/src/__tests__/helpers/ab-summary-fixtures.ts @@ -36,6 +36,7 @@ export function withUsage( costUsd: usage.costUsd, pricingSource: 'runtime', }, + tokenSummarySource: 'final', durationMs: usage.durationMs, }; } diff --git a/packages/headless/src/ab-summary.ts b/packages/headless/src/ab-summary.ts index 115ac5b621..827636a865 100644 --- a/packages/headless/src/ab-summary.ts +++ b/packages/headless/src/ab-summary.ts @@ -1,7 +1,7 @@ import { canonicalJson } from './ab-manifest.js'; import { BUDGET_EXHAUSTED_RUNTIME_UNAVAILABLE_REASON } from './fixed-prompt-controller.js'; import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; -import type { HarborCellTokenSummary } from './cell-output.js'; +import { hasFinalHarborCellTokenSummary, type HarborCellTokenSummary } from './cell-output.js'; import { assertRatio } from './numeric-guards.js'; import type { AbArmSummary, @@ -671,11 +671,7 @@ function summarizeAttemptPairs( function hasCompleteTokenSummary( event: FixedPromptTaskWalEvent, ): event is FixedPromptTaskWalEvent & { tokenSummary: HarborCellTokenSummary } { - if (!hasTokenSummary(event)) return false; - if (event.type === 'task_budget_exhausted') return event.tokenSummarySource === 'final'; - if (event.type !== 'task_completed') return false; - if (event.tokenSummarySource !== undefined) return event.tokenSummarySource === 'final'; - return event.status === 'completed'; + return hasTokenSummary(event) && hasFinalHarborCellTokenSummary(event); } function hasTokenSummary( diff --git a/packages/headless/src/cell-output.ts b/packages/headless/src/cell-output.ts index 723b8f68b7..5f6b232277 100644 --- a/packages/headless/src/cell-output.ts +++ b/packages/headless/src/cell-output.ts @@ -215,6 +215,13 @@ export interface HarborCellOutput { runtimeRefs: HarborCellRuntimeRefs; } +export function hasFinalHarborCellTokenSummary(value: { + tokenSummary?: HarborCellTokenSummary; + tokenSummarySource?: HarborCellOutput['tokenSummarySource']; +}): value is { tokenSummary: HarborCellTokenSummary; tokenSummarySource: 'final' } { + return value.tokenSummary !== undefined && value.tokenSummarySource === 'final'; +} + export function buildHarborCellOutput(input: { invocation: InvocationResult; runtimeEventsPath: string; diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index aecd313fd3..290bd2fa59 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { appendFile, mkdir, readFile, truncate, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; import { + hasFinalHarborCellTokenSummary, validateHarborCellOutput, type HarborCellExecutionIdentity, type HarborCellOutput, @@ -830,6 +831,9 @@ function taskPlumbingFailedEvent(input: { expectedPromptHash: input.expectedPromptHash, runtimeRefs: input.output.cell.runtimeRefs, ...(input.output.cell.tokenSummary ? { tokenSummary: input.output.cell.tokenSummary } : {}), + ...(input.output.cell.tokenSummarySource + ? { tokenSummarySource: input.output.cell.tokenSummarySource } + : {}), ...(input.output.cell.contextBudgetPolicy ? { contextBudgetPolicy: input.output.cell.contextBudgetPolicy } : {}), @@ -892,7 +896,7 @@ function classifyPlumbingFailure( error: `Harbor cell prompt hash ${promptHash} did not match ${expectedPromptHash}`, }; } - if (requireFinalUsage && output.cell.tokenSummary === undefined) { + if (requireFinalUsage && !hasFinalHarborCellTokenSummary(output.cell)) { return { errorClass: 'missing_token_usage', error: 'Harbor cell did not report final token usage', @@ -1094,8 +1098,7 @@ function taskBudgetExhaustedEvent(input: { const tokenSummary = artifactRefs.cellOutput?.tokenSummary ?? artifactRefs.tokenSummary; const tokenSummarySource = tokenSummary ? artifactRefs.cellOutput - ? (artifactRefs.cellOutput.tokenSummarySource ?? - (artifactRefs.cellOutput.status === 'completed' ? 'final' : 'checkpoint')) + ? (artifactRefs.cellOutput.tokenSummarySource ?? 'checkpoint') : 'checkpoint' : undefined; const executionIdentity = diff --git a/packages/headless/src/fixed-prompt-wal-types.ts b/packages/headless/src/fixed-prompt-wal-types.ts index 6683617baa..0ca2595d2d 100644 --- a/packages/headless/src/fixed-prompt-wal-types.ts +++ b/packages/headless/src/fixed-prompt-wal-types.ts @@ -179,6 +179,7 @@ export interface FixedPromptTaskPlumbingFailedEvent { expectedPromptHash?: string; runtimeRefs?: HarborCellRuntimeRefs; tokenSummary?: HarborCellTokenSummary; + tokenSummarySource?: 'final' | 'checkpoint'; contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; contextBudgetSummary?: HarborCellContextBudgetSummary; continuationSummary?: HarborCellContinuationSummary; From a15404272bb4ded59c626291fad63ee811ef2629 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 11:24:15 +0800 Subject: [PATCH 05/10] fix(headless): derive usage provenance at producers --- packages/headless/harbor/codex_agent.py | 9 ++++- packages/headless/harbor/opencode_agent.py | 11 +++++- .../src/__tests__/harbor-adapter.test.ts | 11 +++++- .../src/__tests__/harbor-task-runner.test.ts | 26 ++++++++++---- .../src/__tests__/pier-task-runner.test.ts | 9 +++-- packages/headless/src/harbor-task-runner.ts | 36 ++++++++++++++----- packages/headless/src/pier-task-runner.ts | 12 ++++--- 7 files changed, 89 insertions(+), 25 deletions(-) diff --git a/packages/headless/harbor/codex_agent.py b/packages/headless/harbor/codex_agent.py index d9fd9f1125..b1104579dd 100644 --- a/packages/headless/harbor/codex_agent.py +++ b/packages/headless/harbor/codex_agent.py @@ -463,7 +463,14 @@ def _write_cell_output(self, context: AgentContext) -> None: "runtimeEventsPath": "/logs/agent/runtime-events.jsonl", "promptHash": identity["systemPromptHash"], "executionIdentity": identity, - **({"tokenSummary": token_summary} if token_summary is not None else {}), + **( + { + "tokenSummary": token_summary, + "tokenSummarySource": "checkpoint" if failed else "final", + } + if token_summary is not None + else {} + ), "toolSummary": { "providerVisibleToolCount": 0, "actualToolCalls": sum(tool_call_counts.values()), diff --git a/packages/headless/harbor/opencode_agent.py b/packages/headless/harbor/opencode_agent.py index b71a3ce1fe..c140579a5c 100644 --- a/packages/headless/harbor/opencode_agent.py +++ b/packages/headless/harbor/opencode_agent.py @@ -429,7 +429,16 @@ def _write_cell_output(self, context: AgentContext) -> None: "runtimeEventsPath": "/logs/agent/runtime-events.jsonl", "promptHash": prompt_hash, "executionIdentity": execution_identity, - **({"tokenSummary": token_summary} if token_summary is not None else {}), + **( + { + "tokenSummary": token_summary, + "tokenSummarySource": ( + "checkpoint" if hasattr(self, "_failure_class") else "final" + ), + } + if token_summary is not None + else {} + ), "toolSummary": { "providerVisibleToolCount": 0, "actualToolCalls": sum(tool_call_counts.values()), diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index 93194d5217..a0aaed0f11 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -3708,6 +3708,7 @@ try: assert cell["tokenSummary"]["cacheMissInput"] == 60, cell assert cell["tokenSummary"]["cacheWriteInput"] == 10, cell assert cell["tokenSummary"]["reasoning"] == 5, cell + assert cell["tokenSummarySource"] == "final", cell assert cell["toolSummary"]["actualToolCallCounts"] == {"bash": 1}, cell assert "test-zai-key" not in json.dumps(cell), cell @@ -3749,10 +3750,12 @@ try: pass else: raise AssertionError("expected OpenCode failure") - failing_agent.populate_context_post_run(AgentContext()) + failing_agent._parse_stdout = agent._parse_stdout + failing_agent.populate_context_post_run(context) failed_cell = json.loads((Path(tmp) / "maka-cell-output.json").read_text(encoding="utf-8")) assert failed_cell["status"] == "failed", failed_cell assert failed_cell["errorClass"] == "auth", failed_cell + assert failed_cell["tokenSummarySource"] == "checkpoint", failed_cell assert failed_cell["finishedAt"] >= failed_cell["startedAt"], failed_cell print("opencode_estimated_cost_usd", context.cost_usd) finally: @@ -4921,6 +4924,7 @@ with tempfile.TemporaryDirectory() as tmp: assert cell["tokenSummary"]["cacheMissInput"] == 60, cell assert cell["tokenSummary"]["output"] == 25, cell assert abs(cell["tokenSummary"]["costUsd"] - 0.00107) < 1e-12, cell + assert cell["tokenSummarySource"] == "final", cell assert cell["toolSummary"]["actualToolCallCounts"] == {"command_execution": 1}, cell assert "ephemeral-token" not in json.dumps(cell), cell @@ -4934,6 +4938,10 @@ with tempfile.TemporaryDirectory() as tmp: "MAKA_PROVIDER_PROXY_TOKEN": "ephemeral-token", "MAKA_MODEL": "gpt-5.6-sol", "MAKA_SYSTEM_PROMPT": "", + "MAKA_TRIAL_INPUT_USD_PER_1M": "5", + "MAKA_TRIAL_CACHE_READ_USD_PER_1M": "0.5", + "MAKA_TRIAL_OUTPUT_USD_PER_1M": "30", + "MAKA_TRIAL_PRICING_SOURCE": "openai-gpt-5.6-sol-2026-07-20", }, ) try: @@ -4946,6 +4954,7 @@ with tempfile.TemporaryDirectory() as tmp: failed = json.loads((logs / "maka-cell-output.json").read_text(encoding="utf-8")) assert failed["status"] == "failed", failed assert failed["errorClass"] == "auth", failed + assert failed["tokenSummarySource"] == "checkpoint", failed transport = MakaCodexAgent( logs, diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index a6af928be5..f4d2fc04a0 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -928,7 +928,13 @@ describe('createHarborTaskRunner', () => { await withRun(async ({ jobsDir, repo, keyFile }) => { const upstream = createServer((_request, response) => { response.writeHead(200, { 'content-type': 'text/event-stream' }); - response.write('data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'); + response.write( + [ + 'event: message_start', + 'data: {"type":"message_start","message":{"usage":{"input_tokens":70,"cache_creation_input_tokens":10,"cache_read_input_tokens":20,"output_tokens":1}}}', + '', + ].join('\n'), + ); }); await new Promise((resolve) => upstream.listen(0, '127.0.0.1', resolve)); const address = upstream.address(); @@ -937,14 +943,15 @@ describe('createHarborTaskRunner', () => { const runner = createHarborTaskRunner({ makaRepoPath: repo, jobsDir, - agent: 'codex', - codexToolchainPath: '/toolchain', - agentVersion: '0.146.0', + agent: 'claude-code', + claudeCodeToolchainPath: '/toolchain', + agentVersion: CLAUDE_CODE_TOOLCHAIN_SPEC.claudeCode.version, model: 'deepseek/deepseek-v4-flash', provider: 'deepseek', reasoningEffort: 'max', apiKeyFile: keyFile, agentEnv: { MAKA_BASE_URL: `http://127.0.0.1:${address.port}` }, + pricing: { inputUsdPer1M: 1, outputUsdPer1M: 2 }, runHarbor: async (request) => { const proxyUrl = request.env?.MAKA_PROVIDER_PROXY_URL?.replace( 'host.docker.internal', @@ -953,9 +960,9 @@ describe('createHarborTaskRunner', () => { const proxyToken = request.env?.MAKA_PROVIDER_PROXY_TOKEN; assert.ok(proxyUrl && proxyToken); const abort = new AbortController(); - const pending = fetch(`${proxyUrl}/chat/completions`, { + const pending = fetch(`${proxyUrl}/messages`, { method: 'POST', - headers: { authorization: `Bearer ${proxyToken}` }, + headers: { 'x-api-key': proxyToken }, body: '{}', signal: abort.signal, }); @@ -963,13 +970,18 @@ describe('createHarborTaskRunner', () => { assert.equal(response.status, 200); abort.abort(); await response.body?.cancel().catch(() => {}); - return fakeRunner({ reward: '1\n' })(request); + return fakeRunner({ + reward: '1\n', + cell: cellOutput({ tokenSummary: undefined }), + })(request); }, }); const output = await runner(runInput()); assert.equal(output.harbor.reward, 1); + assert.equal(output.cell.tokenSummary?.total, 101); + assert.equal(output.cell.tokenSummarySource, 'checkpoint'); const telemetry = JSON.parse( await readFile(output.cell.providerTelemetryPath!, 'utf8'), ) as { requests: Array<{ outcome: string }> }; diff --git a/packages/headless/src/__tests__/pier-task-runner.test.ts b/packages/headless/src/__tests__/pier-task-runner.test.ts index 6f2502248b..e4c8de0e1a 100644 --- a/packages/headless/src/__tests__/pier-task-runner.test.ts +++ b/packages/headless/src/__tests__/pier-task-runner.test.ts @@ -1865,12 +1865,13 @@ test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is reasoningEffort: 'max', opencodeToolchainPath: repo, apiKeyFile: '/secrets/deepseek.key', + pricing: { inputUsdPer1M: 1, outputUsdPer1M: 2 }, providerProxyHub: { baseUrl: 'http://host.docker.internal:443', issue: () => ({ baseUrl: 'http://host.docker.internal:443', token: 'ephemeral-token', - usage: () => null, + usage: () => ({ input: 10, cacheRead: 0, cacheWrite: 0, output: 5 }), telemetry: () => [ { requestId: 1, @@ -1882,6 +1883,7 @@ test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is bodyChunks: 1, responseBytes: 64, terminalEvent: outcome === 'completed', + usage: { input: 10, cacheRead: 0, cacheWrite: 0, output: 5 }, }, ], close: async () => {}, @@ -1904,9 +1906,12 @@ test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is // A completed terminal request takes the normal reward path. const output = await makeRunner('completed')(runInput()); assert.equal(output.harbor.reward, 0); + assert.equal(output.cell.tokenSummarySource, 'final'); // So does a tail the agent tore down itself on its way out: the trial // raised nothing and the verifier graded it, so the cell is evidence. - assert.equal((await makeRunner('aborted')(runInput())).harbor.reward, 0); + const aborted = await makeRunner('aborted')(runInput()); + assert.equal(aborted.harbor.reward, 0); + assert.equal(aborted.cell.tokenSummarySource, 'checkpoint'); }); }); diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 10f3d00446..5ed337c3d1 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -618,14 +618,12 @@ export function createHarborTaskRunner(options: HarborTaskRunnerOptions): TaskRu selectedUsage && selectedUsage !== rawCell.tokenSummary ? { ...rawCell, tokenSummary: selectedUsage, tokenSummarySource: 'checkpoint' as const } : rawCell; - const usageCell = - checkpointedCell.tokenSummary || !providerUsage || !runnerOptions.pricing - ? checkpointedCell - : { - ...checkpointedCell, - tokenSummary: providerTokenSummary(providerUsage, runnerOptions.pricing), - tokenSummarySource: 'final' as const, - }; + const usageCell = withProviderTokenSummary( + checkpointedCell, + providerUsage, + providerTelemetry, + runnerOptions.pricing, + ); const cell = completeTimedOutTrial ? { ...usageCell, @@ -1519,6 +1517,28 @@ export function providerTokenSummary( }; } +/** Fill a missing cell summary from provider telemetry without overstating + * partial usage as final. Only requests that contributed usage need terminal + * evidence; requests without usage do not affect the aggregate. */ +export function withProviderTokenSummary( + cell: HarborCellOutput, + usage: ProviderTokenUsage | null, + telemetry: readonly ProviderRequestTelemetry[], + pricing: HarborTaskPricing | undefined, +): HarborCellOutput { + if (cell.tokenSummary || !usage || !pricing) return cell; + const measuredRequests = telemetry.filter((request) => request.usage !== undefined); + const tokenSummarySource = + measuredRequests.length > 0 && measuredRequests.every((request) => request.terminalEvent) + ? ('final' as const) + : ('checkpoint' as const); + return { + ...cell, + tokenSummary: providerTokenSummary(usage, pricing), + tokenSummarySource, + }; +} + /** Match the Maka host connection's protocol authority when configuring its auth proxy. */ export function providerProxyApiProtocol( agent: HarborTaskRunnerOptions['agent'], diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index f36dfe932f..f60ef9a3a5 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -22,7 +22,6 @@ import { providerProxyApiProtocol, providerRequiresSecret, providerTelemetryArtifactRefs, - providerTokenSummary, readCellOutput, readTimedOutTrialArtifacts, classifyTrialTermination, @@ -33,6 +32,7 @@ import { withProviderTelemetryArtifact, incompleteTerminalProviderRequest, trialGradeSurvivingProviderOutage, + withProviderTokenSummary, modelForOpenCode, type HarborTaskPricing, } from './harbor-task-runner.js'; @@ -620,10 +620,12 @@ export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner input.task.id, PierInfraError, ); - const cell = - rawCell.tokenSummary || !providerUsage || !options.pricing - ? rawCell - : { ...rawCell, tokenSummary: providerTokenSummary(providerUsage, options.pricing) }; + const cell = withProviderTokenSummary( + rawCell, + providerUsage, + providerTelemetry, + options.pricing, + ); const hostEventsPath = join(trialDir, TRIAL_RUNTIME_EVENTS); // Pier's verifier grading is the scoring authority. Surface it as the // structured verifier outcome the controller requires: without it a From fc30a10515b98a78a08788ce9b972295764994d8 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 11:36:04 +0800 Subject: [PATCH 06/10] fix(headless): keep incomplete usage provisional --- packages/headless/harbor/maka_agent.py | 1 + .../src/__tests__/harbor-adapter.test.ts | 5 +++- .../src/__tests__/harbor-task-runner.test.ts | 28 +++++++++++++++++++ packages/headless/src/harbor-task-runner.ts | 7 +++-- packages/headless/src/provider-auth-proxy.ts | 3 ++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index ca0a8fe711..1d93b1f7e6 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -710,6 +710,7 @@ def _read_cell_output(self, *, required: bool) -> dict[str, Any] | None: checkpoint = json.loads(checkpoint_path.read_text(encoding="utf-8")) if isinstance(checkpoint, dict): output["tokenSummary"] = checkpoint + output["tokenSummarySource"] = "checkpoint" output_path.write_text(f"{json.dumps(output, indent=2)}\n", encoding="utf-8") except (OSError, json.JSONDecodeError) as exc: self.logger.debug("Could not hydrate Maka deadline usage from %s: %s", checkpoint_path, exc) diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index a0aaed0f11..d6d033acd9 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -1984,7 +1984,10 @@ with tempfile.TemporaryDirectory() as tmp: (Path(tmp) / "maka-cell-usage-checkpoint.json").write_text(json.dumps(deadline_usage), encoding="utf-8") hydrated_deadline_output = agent._read_cell_output(required=True) assert hydrated_deadline_output["tokenSummary"] == deadline_usage, hydrated_deadline_output - assert json.loads(deadline_output_path.read_text(encoding="utf-8"))["tokenSummary"] == deadline_usage + assert hydrated_deadline_output["tokenSummarySource"] == "checkpoint", hydrated_deadline_output + persisted_deadline_output = json.loads(deadline_output_path.read_text(encoding="utf-8")) + assert persisted_deadline_output["tokenSummary"] == deadline_usage, persisted_deadline_output + assert persisted_deadline_output["tokenSummarySource"] == "checkpoint", persisted_deadline_output class DownloadEnvironment: def __init__(self): diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index f4d2fc04a0..0a91fea77a 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -23,6 +23,7 @@ import { HarborInfraError, incompleteTerminalProviderRequest, trialGradeSurvivingProviderOutage, + withProviderTokenSummary, MAKA_SETTLEMENT_GRACE_SEC, type HarborProcessRunner, type HarborRunRequest, @@ -994,6 +995,33 @@ describe('createHarborTaskRunner', () => { }); }); + test('keeps provider usage provisional when an earlier generation request has no usage', () => { + const usage = { input: 10, cacheRead: 0, cacheWrite: 0, output: 5 }; + const request = (overrides: Partial): ProviderRequestTelemetry => ({ + requestId: 1, + method: 'POST', + path: '/v1/messages', + protocol: 'anthropic-sse', + status: 200, + outcome: 'completed', + durationMs: 5, + bodyChunks: 1, + responseBytes: 64, + usageStream: true, + terminalEvent: true, + ...overrides, + }); + + const output = withProviderTokenSummary( + cellOutput({ tokenSummary: undefined }), + usage, + [request({ outcome: 'aborted', terminalEvent: false }), request({ requestId: 2, usage })], + { inputUsdPer1M: 1, outputUsdPer1M: 2 }, + ); + + assert.equal(output.tokenSummarySource, 'checkpoint'); + }); + test('gives Codex an ephemeral OpenAI proxy without exposing the provider key file', async () => { await withRun(async ({ jobsDir, repo, keyFile }) => { const captured: { config?: Record } = {}; diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 5ed337c3d1..dcc300f05c 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -1527,9 +1527,12 @@ export function withProviderTokenSummary( pricing: HarborTaskPricing | undefined, ): HarborCellOutput { if (cell.tokenSummary || !usage || !pricing) return cell; - const measuredRequests = telemetry.filter((request) => request.usage !== undefined); + const usageRequests = telemetry.filter( + (request) => request.usageStream === true || request.usage !== undefined, + ); const tokenSummarySource = - measuredRequests.length > 0 && measuredRequests.every((request) => request.terminalEvent) + usageRequests.length > 0 && + usageRequests.every((request) => request.usage !== undefined && request.terminalEvent) ? ('final' as const) : ('checkpoint' as const); return { diff --git a/packages/headless/src/provider-auth-proxy.ts b/packages/headless/src/provider-auth-proxy.ts index 3baf1cbd2e..8a58bd84b0 100644 --- a/packages/headless/src/provider-auth-proxy.ts +++ b/packages/headless/src/provider-auth-proxy.ts @@ -73,6 +73,8 @@ export interface ProviderRequestTelemetry { durationMs: number; bodyChunks: number; responseBytes: number; + /** The upstream response was an SSE stream parsed for provider usage. */ + usageStream?: boolean; terminalEvent: boolean; usage?: ProviderTokenUsage; errorClass?: string; @@ -488,6 +490,7 @@ async function forwardProviderRequest(input: { upstreamResponse.headers.get('content-type')?.includes('text/event-stream') ? new SseUsageParser(input.usageProtocol) : null; + if (responseUsage) requestTelemetry.usageStream = true; if (upstreamResponse.body) { for await (const chunk of upstreamResponse.body) { const observedAt = input.now(); From c9ceda31228d62fa87a700ff68fe9377d8a33043 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 11:46:38 +0800 Subject: [PATCH 07/10] fix(headless): downgrade unsettled native usage --- .../src/__tests__/harbor-task-runner.test.ts | 33 +++++++++++++------ .../src/__tests__/pier-task-runner.test.ts | 19 +++++++++-- packages/headless/src/harbor-task-runner.ts | 24 ++++++++++++-- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index 0a91fea77a..a358749410 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -973,7 +973,7 @@ describe('createHarborTaskRunner', () => { await response.body?.cancel().catch(() => {}); return fakeRunner({ reward: '1\n', - cell: cellOutput({ tokenSummary: undefined }), + cell: cellOutput({ tokenSummarySource: 'final' }), })(request); }, }); @@ -981,7 +981,7 @@ describe('createHarborTaskRunner', () => { const output = await runner(runInput()); assert.equal(output.harbor.reward, 1); - assert.equal(output.cell.tokenSummary?.total, 101); + assert.equal(output.cell.tokenSummary?.total, 150); assert.equal(output.cell.tokenSummarySource, 'checkpoint'); const telemetry = JSON.parse( await readFile(output.cell.providerTelemetryPath!, 'utf8'), @@ -995,7 +995,7 @@ describe('createHarborTaskRunner', () => { }); }); - test('keeps provider usage provisional when an earlier generation request has no usage', () => { + test('keeps provider usage provisional when an earlier request cannot prove final usage', () => { const usage = { input: 10, cacheRead: 0, cacheWrite: 0, output: 5 }; const request = (overrides: Partial): ProviderRequestTelemetry => ({ requestId: 1, @@ -1012,14 +1012,27 @@ describe('createHarborTaskRunner', () => { ...overrides, }); - const output = withProviderTokenSummary( - cellOutput({ tokenSummary: undefined }), - usage, - [request({ outcome: 'aborted', terminalEvent: false }), request({ requestId: 2, usage })], - { inputUsdPer1M: 1, outputUsdPer1M: 2 }, - ); + for (const incomplete of [ + request({ outcome: 'aborted', terminalEvent: false }), + request({ + outcome: 'aborted', + status: undefined, + bodyChunks: 0, + responseBytes: 0, + usageStream: undefined, + terminalEvent: false, + upstreamStartMs: 1, + }), + ]) { + const output = withProviderTokenSummary( + cellOutput({ tokenSummary: undefined }), + usage, + [incomplete, request({ requestId: 2, usage })], + { inputUsdPer1M: 1, outputUsdPer1M: 2 }, + ); - assert.equal(output.tokenSummarySource, 'checkpoint'); + assert.equal(output.tokenSummarySource, 'checkpoint'); + } }); test('gives Codex an ephemeral OpenAI proxy without exposing the provider key file', async () => { diff --git a/packages/headless/src/__tests__/pier-task-runner.test.ts b/packages/headless/src/__tests__/pier-task-runner.test.ts index e4c8de0e1a..f5d0956659 100644 --- a/packages/headless/src/__tests__/pier-task-runner.test.ts +++ b/packages/headless/src/__tests__/pier-task-runner.test.ts @@ -6,6 +6,7 @@ import { createServer, type AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; +import { tokenSummary } from './helpers/cell-output-fixtures.js'; import type { HarborCellOutput } from '../cell-output.js'; import { FixedPromptBudgetExhaustedError, @@ -1852,7 +1853,7 @@ test('createPierTaskRunner requires the OpenCode toolchain mount for the OpenCod test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is incomplete', async () => { await withDirs(async ({ jobsDir, repo }) => { - const makeRunner = (outcome: ProviderRequestTelemetry['outcome']) => + const makeRunner = (outcome: ProviderRequestTelemetry['outcome'], cell?: HarborCellOutput) => createPierTaskRunner( baseOptions({ jobsDir, @@ -1890,7 +1891,7 @@ test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is }), close: async () => {}, }, - runPier: fakePier({ reward: 0 }), + runPier: fakePier({ reward: 0, ...(cell ? { cell } : {}) }), }), ); @@ -1912,6 +1913,20 @@ test('createPierTaskRunner rejects a Pier cell whose terminal provider stream is const aborted = await makeRunner('aborted')(runInput()); assert.equal(aborted.harbor.reward, 0); assert.equal(aborted.cell.tokenSummarySource, 'checkpoint'); + const nativeAborted = await makeRunner( + 'aborted', + cellOutput({ + tokenSummary: tokenSummary({ + input: 10, + output: 5, + reasoning: 0, + total: 15, + costUsd: 0.00002, + }), + tokenSummarySource: 'final', + }), + )(runInput()); + assert.equal(nativeAborted.cell.tokenSummarySource, 'checkpoint'); }); }); diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index dcc300f05c..6a0a8a76b3 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -1526,10 +1526,15 @@ export function withProviderTokenSummary( telemetry: readonly ProviderRequestTelemetry[], pricing: HarborTaskPricing | undefined, ): HarborCellOutput { + const usageRequests = telemetry.filter(providerRequestMayHaveUsage); + if ( + cell.tokenSummary && + cell.tokenSummarySource === 'final' && + usageRequests.some((request) => !request.terminalEvent) + ) { + return { ...cell, tokenSummarySource: 'checkpoint' }; + } if (cell.tokenSummary || !usage || !pricing) return cell; - const usageRequests = telemetry.filter( - (request) => request.usageStream === true || request.usage !== undefined, - ); const tokenSummarySource = usageRequests.length > 0 && usageRequests.every((request) => request.usage !== undefined && request.terminalEvent) @@ -1542,6 +1547,19 @@ export function withProviderTokenSummary( }; } +function providerRequestMayHaveUsage(request: ProviderRequestTelemetry): boolean { + if (request.usageStream === true || request.usage !== undefined) return true; + const method = request.method.toUpperCase(); + return ( + request.protocol !== undefined && + method !== 'GET' && + method !== 'HEAD' && + request.upstreamStartMs !== undefined && + request.status === undefined && + request.outcome !== 'completed' + ); +} + /** Match the Maka host connection's protocol authority when configuring its auth proxy. */ export function providerProxyApiProtocol( agent: HarborTaskRunnerOptions['agent'], From eaaa18e85b1767dcb83717996d3bed1faad61ba8 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 12:11:48 +0800 Subject: [PATCH 08/10] test(headless): mark harness fixtures as final --- .../headless/src/__tests__/harness-ab-run.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/headless/src/__tests__/harness-ab-run.test.ts b/packages/headless/src/__tests__/harness-ab-run.test.ts index f084e5701b..07fdd6c1df 100644 --- a/packages/headless/src/__tests__/harness-ab-run.test.ts +++ b/packages/headless/src/__tests__/harness-ab-run.test.ts @@ -488,7 +488,11 @@ describe('runHarnessAbComparison', () => { const meteredRunner = maka.harborRunner; maka.harborRunner = async (input) => { const output = await meteredRunner(input); - const { tokenSummary: _tokenSummary, ...cell } = output.cell; + const { + tokenSummary: _tokenSummary, + tokenSummarySource: _tokenSummarySource, + ...cell + } = output.cell; return { ...output, cell }; }; @@ -654,7 +658,11 @@ describe('runHarnessArmCohort', () => { const meteredRunner = codex.harborRunner; codex.harborRunner = async (input) => { const output = await meteredRunner(input); - const { tokenSummary: _tokenSummary, ...cell } = output.cell; + const { + tokenSummary: _tokenSummary, + tokenSummarySource: _tokenSummarySource, + ...cell + } = output.cell; return { ...output, cell }; }; @@ -711,6 +719,7 @@ function harnessArm(id: HarnessAbArmId, calls: string[], beforeRun?: () => Promi total: 110, costUsd: 0.000184, }), + tokenSummarySource: 'final', toolSummary: { providerVisibleToolCount: 1, actualToolCalls: 1, From 1a5fd14c91a5c2f93ef0d5cf232cb8122f9f3281 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 14:01:51 +0800 Subject: [PATCH 09/10] refactor(headless): clarify usage reconciliation --- .../headless/src/__tests__/harbor-task-runner.test.ts | 4 ++-- packages/headless/src/harbor-task-runner.ts | 10 ++++------ packages/headless/src/pier-task-runner.ts | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/headless/src/__tests__/harbor-task-runner.test.ts b/packages/headless/src/__tests__/harbor-task-runner.test.ts index a358749410..cb98fc14b3 100644 --- a/packages/headless/src/__tests__/harbor-task-runner.test.ts +++ b/packages/headless/src/__tests__/harbor-task-runner.test.ts @@ -23,7 +23,7 @@ import { HarborInfraError, incompleteTerminalProviderRequest, trialGradeSurvivingProviderOutage, - withProviderTokenSummary, + reconcileProviderTokenSummary, MAKA_SETTLEMENT_GRACE_SEC, type HarborProcessRunner, type HarborRunRequest, @@ -1024,7 +1024,7 @@ describe('createHarborTaskRunner', () => { upstreamStartMs: 1, }), ]) { - const output = withProviderTokenSummary( + const output = reconcileProviderTokenSummary( cellOutput({ tokenSummary: undefined }), usage, [incomplete, request({ requestId: 2, usage })], diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 6a0a8a76b3..ad613da1c6 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -618,7 +618,7 @@ export function createHarborTaskRunner(options: HarborTaskRunnerOptions): TaskRu selectedUsage && selectedUsage !== rawCell.tokenSummary ? { ...rawCell, tokenSummary: selectedUsage, tokenSummarySource: 'checkpoint' as const } : rawCell; - const usageCell = withProviderTokenSummary( + const usageCell = reconcileProviderTokenSummary( checkpointedCell, providerUsage, providerTelemetry, @@ -1491,7 +1491,7 @@ function usesHostProviderProxy( } /** Shared cost math across runners: build the cell token summary from proxy-observed usage and per-1M pricing. */ -export function providerTokenSummary( +function providerTokenSummary( usage: ProviderTokenUsage, pricing: HarborTaskPricing, ): NonNullable { @@ -1517,10 +1517,8 @@ export function providerTokenSummary( }; } -/** Fill a missing cell summary from provider telemetry without overstating - * partial usage as final. Only requests that contributed usage need terminal - * evidence; requests without usage do not affect the aggregate. */ -export function withProviderTokenSummary( +/** Reconcile native and proxy usage without labeling incomplete provider evidence as final. */ +export function reconcileProviderTokenSummary( cell: HarborCellOutput, usage: ProviderTokenUsage | null, telemetry: readonly ProviderRequestTelemetry[], diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index f60ef9a3a5..5e1dc5f05a 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -32,7 +32,7 @@ import { withProviderTelemetryArtifact, incompleteTerminalProviderRequest, trialGradeSurvivingProviderOutage, - withProviderTokenSummary, + reconcileProviderTokenSummary, modelForOpenCode, type HarborTaskPricing, } from './harbor-task-runner.js'; @@ -620,7 +620,7 @@ export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner input.task.id, PierInfraError, ); - const cell = withProviderTokenSummary( + const cell = reconcileProviderTokenSummary( rawCell, providerUsage, providerTelemetry, From 5e56e9968ce7d2831345d2a55061885a8cb81885 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Sun, 9 Aug 2026 14:44:32 +0800 Subject: [PATCH 10/10] fix(headless): retain error response usage --- .../src/__tests__/provider-auth-proxy.test.ts | 43 +++++++++++++------ packages/headless/src/provider-auth-proxy.ts | 2 +- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/headless/src/__tests__/provider-auth-proxy.test.ts b/packages/headless/src/__tests__/provider-auth-proxy.test.ts index 6565521cf3..6e84ebd44b 100644 --- a/packages/headless/src/__tests__/provider-auth-proxy.test.ts +++ b/packages/headless/src/__tests__/provider-auth-proxy.test.ts @@ -496,7 +496,7 @@ test('provider auth proxy accepts a client x-api-key while authenticating upstre } }); -test('provider auth proxy totals Anthropic streaming usage without changing the response bytes', async () => { +test('provider auth proxy totals Anthropic usage across success and error streams', async () => { const dir = await mkdtemp(join(tmpdir(), 'maka-provider-proxy-usage-')); const stream = [ 'event: message_start', @@ -505,9 +505,14 @@ test('provider auth proxy totals Anthropic streaming usage without changing the 'event: message_delta', 'data: {"type":"message_delta","usage":{"output_tokens":25}}', '', + 'event: message_stop', + 'data: {"type":"message_stop"}', + '', ].join('\n'); + const statuses = [500, 200]; + let requestIndex = 0; const upstream = createServer((_request, response) => { - response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.writeHead(statuses[requestIndex++] ?? 500, { 'content-type': 'text/event-stream' }); response.write(stream.slice(0, 91)); response.end(stream.slice(91)); }); @@ -524,18 +529,32 @@ test('provider auth proxy totals Anthropic streaming usage without changing the }); try { - const response = await fetch(`${proxy.baseUrl}/messages`, { - method: 'POST', - headers: { authorization: `Bearer ${proxy.token}` }, - body: '{}', - }); - assert.equal(await response.text(), stream); + for (const status of statuses) { + const response = await fetch(`${proxy.baseUrl}/messages`, { + method: 'POST', + headers: { authorization: `Bearer ${proxy.token}` }, + body: '{}', + }); + assert.equal(response.status, status); + assert.equal(await response.text(), stream); + } assert.deepEqual(proxy.usage(), { - input: 100, - cacheRead: 20, - cacheWrite: 10, - output: 25, + input: 200, + cacheRead: 40, + cacheWrite: 20, + output: 50, }); + assert.deepEqual( + proxy.telemetry().map(({ status, outcome, terminalEvent }) => ({ + status, + outcome, + terminalEvent, + })), + [ + { status: 500, outcome: 'failed', terminalEvent: true }, + { status: 200, outcome: 'completed', terminalEvent: true }, + ], + ); } finally { await proxy.close(); await new Promise((resolve, reject) => diff --git a/packages/headless/src/provider-auth-proxy.ts b/packages/headless/src/provider-auth-proxy.ts index 8a58bd84b0..5d100a022c 100644 --- a/packages/headless/src/provider-auth-proxy.ts +++ b/packages/headless/src/provider-auth-proxy.ts @@ -525,7 +525,7 @@ async function forwardProviderRequest(input: { } } const parsed = responseUsage?.finish() ?? null; - if (upstreamResponse.ok && parsed?.usage) input.usage.add(parsed.usage); + if (parsed?.usage) input.usage.add(parsed.usage); requestTelemetry.usage = parsed?.usage ?? undefined; requestTelemetry.terminalEvent = parsed?.terminalEvent ?? false; requestTelemetry.outcome = !upstreamResponse.ok