fix(headless): preserve benchmark evidence provenance

packages/core/src/__tests__/fixtures/runtime-event-validation-corpus.json

@@ -78,9 +78,12 @@
           "traceEventId": "trace-1",
           "toolCallId": "tool-call-1",
           "providerEventId": "provider-event-1",
+          "sourceMessageDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
           "providerRequestTraceId": "provider-trace-1",
           "artifactId": "artifact-1",
           "operationId": "operation-1",
+          "parentToolCallId": "parent-tool-call-1",
+          "parentOperationId": "parent-operation-1",
           "stepId": "step-1",
           "sourceInvocationId": "source-invocation-1",
           "sourceRunId": "source-run-1",

packages/headless/harbor/codex_agent.py

@@ -463,7 +463,14 @@ def _write_cell_output(self, context: AgentContext) -> None:
             "runtimeEventsPath": "/logs/agent/runtime-events.jsonl",
             "promptHash": identity["systemPromptHash"],
             "executionIdentity": identity,
-            **({"tokenSummary": token_summary} if token_summary is not None else {}),
+            **(
+                {
+                    "tokenSummary": token_summary,
+                    "tokenSummarySource": "checkpoint" if failed else "final",
+                }
+                if token_summary is not None
+                else {}
+            ),
             "toolSummary": {
                 "providerVisibleToolCount": 0,
                 "actualToolCalls": sum(tool_call_counts.values()),

packages/headless/harbor/maka_agent.py

@@ -710,6 +710,7 @@ def _read_cell_output(self, *, required: bool) -> dict[str, Any] | None:
                 checkpoint = json.loads(checkpoint_path.read_text(encoding="utf-8"))
                 if isinstance(checkpoint, dict):
                     output["tokenSummary"] = checkpoint
+                    output["tokenSummarySource"] = "checkpoint"
                     output_path.write_text(f"{json.dumps(output, indent=2)}\n", encoding="utf-8")
             except (OSError, json.JSONDecodeError) as exc:
                 self.logger.debug("Could not hydrate Maka deadline usage from %s: %s", checkpoint_path, exc)

packages/headless/harbor/maka_trajectory.py

@@ -163,9 +163,12 @@ class _IncompleteTrajectoryEvidence(ValueError):
     "traceEventId",
     "toolCallId",
     "providerEventId",
+    "sourceMessageDigest",
     "providerRequestTraceId",
     "artifactId",
     "operationId",
+    "parentToolCallId",
+    "parentOperationId",
     "stepId",
     "sourceInvocationId",
     "sourceRunId",
@@ -1786,6 +1789,10 @@ def _is_runtime_refs(refs: Any) -> bool:
     string_refs = _RUNTIME_EVENT_REF_KEYS - {"sourceRuntimeEventHighWater"}
     if any(key in refs and not isinstance(refs[key], str) for key in string_refs):
         return False
+    if "sourceMessageDigest" in refs and re.fullmatch(
+        r"sha256:[0-9a-f]{64}", refs["sourceMessageDigest"]
+    ) is None:
+        return False
     if "sourceRuntimeEventHighWater" not in refs:
         return True
     return _is_nonnegative_safe_integer(refs["sourceRuntimeEventHighWater"])

packages/headless/harbor/opencode_agent.py

@@ -429,7 +429,16 @@ def _write_cell_output(self, context: AgentContext) -> None:
             "runtimeEventsPath": "/logs/agent/runtime-events.jsonl",
             "promptHash": prompt_hash,
             "executionIdentity": execution_identity,
-            **({"tokenSummary": token_summary} if token_summary is not None else {}),
+            **(
+                {
+                    "tokenSummary": token_summary,
+                    "tokenSummarySource": (
+                        "checkpoint" if hasattr(self, "_failure_class") else "final"
+                    ),
+                }
+                if token_summary is not None
+                else {}
+            ),
             "toolSummary": {
                 "providerVisibleToolCount": 0,
                 "actualToolCalls": sum(tool_call_counts.values()),

packages/headless/src/__tests__/cell-output.test.ts

@@ -171,6 +171,7 @@ describe('Harbor cell output contract', () => {
         costUsd: 0.00523,
         pricingSource: 'runtime',
       },
+      tokenSummarySource: 'final',
       toolSummary: {
         providerVisibleToolCount: 6,
         actualToolCalls: 2,
@@ -407,6 +408,7 @@ describe('Harbor cell output contract', () => {
     });
 
     assert.equal(output.steps, 2);
+    assert.equal(output.tokenSummarySource, 'checkpoint');
   });
 
   test('summarizes context budget diagnostics from token usage events', () => {

packages/headless/src/__tests__/fixed-prompt-controller.test.ts

@@ -39,19 +39,27 @@ describe('fixed prompt controller', () => {
         outcome: 'passed' as const,
         attempts: [{ attempt: 1, classification: 'passed' as const, durationMs: 12, reward: 1 }],
       };
+      let clock = 100;
       const result = await runFixedPromptController({
         runId: 'run-1',
         roundId: 'round-1',
         config,
         systemPromptPath,
         resultsJsonlPath: join(dir, 'results.jsonl'),
         tasks: [{ id: 'task-a', path: '/bench/task-a' }],
-        taskRunner: async () => harborOutput({ taskId: 'task-a', verifier }),
+        taskRunner: async () => {
+          clock = 250;
+          return harborOutput({ taskId: 'task-a', verifier, tokenSummarySource: 'final' });
+        },
+        now: () => clock,
       });
 
-      assert.equal(result.events[0]?.type, 'task_completed');
-      if (result.events[0]?.type === 'task_completed')
-        assert.deepEqual(result.events[0].harbor.verifier, verifier);
+      const event = result.events[0];
+      assert.equal(event?.type, 'task_completed');
+      if (event?.type !== 'task_completed') assert.fail('expected completed event');
+      assert.deepEqual(event.harbor.verifier, verifier);
+      assert.equal(event.tokenSummarySource, 'final');
+      assert.equal(event.ts, 250);
     });
   });
 
@@ -1200,6 +1208,7 @@ describe('fixed prompt controller', () => {
       const retainedContextBudgetSummary = contextBudgetSummary({ prunedToolResults: 2 });
       const cell = harborOutput({
         taskId: 'task-a',
+        tokenSummarySource: 'final',
         contextBudgetPolicy: { enabled: true, minRecentTurns: 2 },
         contextBudgetSummary: retainedContextBudgetSummary,
         executionIdentity: {
@@ -1249,6 +1258,35 @@ describe('fixed prompt controller', () => {
     });
   });
 
+  test('keeps legacy completed timeout usage provisional without provenance', async () => {
+    await withDir(async (dir) => {
+      const systemPromptPath = join(dir, 'system_prompt.md');
+      await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
+      const cell = harborOutput({ taskId: 'task-a' }).cell;
+
+      const result = await runFixedPromptController({
+        runId: 'run-1',
+        roundId: 'round-1',
+        config,
+        systemPromptPath,
+        resultsJsonlPath: join(dir, 'results.jsonl'),
+        tasks: [{ id: 'task-a', path: '/bench/task-a' }],
+        taskRunner: async () => {
+          throw new FixedPromptBudgetExhaustedError('agent timed out', undefined, {
+            cellOutput: cell,
+          });
+        },
+        now: () => 100,
+        newId: idFactory(),
+      });
+
+      const event = result.events[0];
+      assert.equal(event?.type, 'task_budget_exhausted');
+      if (event?.type !== 'task_budget_exhausted') assert.fail('expected budget exhaustion event');
+      assert.equal(event.tokenSummarySource, 'checkpoint');
+    });
+  });
+
   test('keeps an early-attested timeout eligible without claiming complete usage', async () => {
     await withDir(async (dir) => {
       const systemPromptPath = join(dir, 'system_prompt.md');
@@ -1300,6 +1338,7 @@ describe('fixed prompt controller', () => {
         taskId: 'task-a',
         status: 'failed',
         errorClass: 'auth',
+        tokenSummarySource: 'checkpoint',
         executionIdentity: {
           llmConnectionSlug: 'fake',
           model: 'fake-model',
@@ -1331,6 +1370,7 @@ describe('fixed prompt controller', () => {
       if (event?.type !== 'task_budget_exhausted') assert.fail('expected budget exhaustion event');
       assert.equal(event.eligible, false);
       assert.equal(event.evidenceErrorClass, 'auth');
+      assert.equal(event.tokenSummarySource, 'checkpoint');
       assert.equal(result.stopReason, 'systemic_provider_failure');
     });
   });
@@ -2283,6 +2323,7 @@ describe('fixed prompt controller', () => {
             reward: 0,
             status: 'failed',
             errorClass: 'aborted',
+            tokenSummarySource: 'final',
             deadlineSettlement: { source: 'benchmark.deadline', mode: 'immediate' },
             verifier: {
               outcome: 'failed',
@@ -2571,6 +2612,45 @@ describe('fixed prompt controller', () => {
     });
   });
 
+  test('rejects checkpoint usage when final usage is required', async () => {
+    await withDir(async (dir) => {
+      const systemPromptPath = join(dir, 'system_prompt.md');
+      await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
+
+      const result = await runFixedPromptController({
+        runId: 'run-1',
+        roundId: 'round-1',
+        config,
+        systemPromptPath,
+        resultsJsonlPath: join(dir, 'results.jsonl'),
+        tasks: [{ id: 'task-a', path: '/bench/task-a' }],
+        requireExecutionIdentity: true,
+        requireFinalUsage: true,
+        expectedPricingProfile: 'test-profile',
+        taskRunner: async () =>
+          harborOutput({
+            taskId: 'task-a',
+            tokenSummarySource: 'checkpoint',
+            executionIdentity: {
+              llmConnectionSlug: 'fake',
+              model: 'fake-model',
+              systemPromptHash: hashSystemPrompt('fixed prompt\n'),
+              pricingProfile: 'test-profile',
+            },
+          }),
+        now: () => 100,
+        newId: idFactory(),
+      });
+
+      const event = result.events[0];
+      assert.equal(event?.type, 'task_plumbing_failed');
+      if (event?.type !== 'task_plumbing_failed') assert.fail('expected plumbing failure event');
+      assert.equal(event.errorClass, 'missing_token_usage');
+      assert.ok(event.tokenSummary);
+      assert.equal(event.tokenSummarySource, 'checkpoint');
+    });
+  });
+
   test('rejects a verifier-graded failed result when required final usage is missing', async () => {
     await withDir(async (dir) => {
       const systemPromptPath = join(dir, 'system_prompt.md');
@@ -3593,6 +3673,7 @@ function harborOutput(input: {
   promptHash?: string;
   omitPromptHash?: boolean;
   tokenSummary?: TaskRunOutput['cell']['tokenSummary'];
+  tokenSummarySource?: 'final' | 'checkpoint';
   omitTokenSummary?: boolean;
   contextBudgetPolicy?: TaskRunOutput['cell']['contextBudgetPolicy'];
   contextBudgetSummary?: TaskRunOutput['cell']['contextBudgetSummary'];
@@ -3636,6 +3717,7 @@ function harborOutput(input: {
             tokenSummary:
               input.tokenSummary ??
               tokenSummary({ input: 1, output: 2, reasoning: 0, total: 3, costUsd: 0.02 }),
+            ...(input.tokenSummarySource ? { tokenSummarySource: input.tokenSummarySource } : {}),
           }),
       ...(input.contextBudgetPolicy ? { contextBudgetPolicy: input.contextBudgetPolicy } : {}),
       ...(input.contextBudgetSummary ? { contextBudgetSummary: input.contextBudgetSummary } : {}),

packages/headless/src/__tests__/harbor-adapter.test.ts

@@ -1984,7 +1984,10 @@ with tempfile.TemporaryDirectory() as tmp:
     (Path(tmp) / "maka-cell-usage-checkpoint.json").write_text(json.dumps(deadline_usage), encoding="utf-8")
     hydrated_deadline_output = agent._read_cell_output(required=True)
     assert hydrated_deadline_output["tokenSummary"] == deadline_usage, hydrated_deadline_output
-    assert json.loads(deadline_output_path.read_text(encoding="utf-8"))["tokenSummary"] == deadline_usage
+    assert hydrated_deadline_output["tokenSummarySource"] == "checkpoint", hydrated_deadline_output
+    persisted_deadline_output = json.loads(deadline_output_path.read_text(encoding="utf-8"))
+    assert persisted_deadline_output["tokenSummary"] == deadline_usage, persisted_deadline_output
+    assert persisted_deadline_output["tokenSummarySource"] == "checkpoint", persisted_deadline_output
 
     class DownloadEnvironment:
         def __init__(self):
@@ -3708,6 +3711,7 @@ try:
         assert cell["tokenSummary"]["cacheMissInput"] == 60, cell
         assert cell["tokenSummary"]["cacheWriteInput"] == 10, cell
         assert cell["tokenSummary"]["reasoning"] == 5, cell
+        assert cell["tokenSummarySource"] == "final", cell
         assert cell["toolSummary"]["actualToolCallCounts"] == {"bash": 1}, cell
         assert "test-zai-key" not in json.dumps(cell), cell
 
@@ -3749,10 +3753,12 @@ try:
             pass
         else:
             raise AssertionError("expected OpenCode failure")
-        failing_agent.populate_context_post_run(AgentContext())
+        failing_agent._parse_stdout = agent._parse_stdout
+        failing_agent.populate_context_post_run(context)
         failed_cell = json.loads((Path(tmp) / "maka-cell-output.json").read_text(encoding="utf-8"))
         assert failed_cell["status"] == "failed", failed_cell
         assert failed_cell["errorClass"] == "auth", failed_cell
+        assert failed_cell["tokenSummarySource"] == "checkpoint", failed_cell
         assert failed_cell["finishedAt"] >= failed_cell["startedAt"], failed_cell
         print("opencode_estimated_cost_usd", context.cost_usd)
 finally:
@@ -4921,6 +4927,7 @@ with tempfile.TemporaryDirectory() as tmp:
     assert cell["tokenSummary"]["cacheMissInput"] == 60, cell
     assert cell["tokenSummary"]["output"] == 25, cell
     assert abs(cell["tokenSummary"]["costUsd"] - 0.00107) < 1e-12, cell
+    assert cell["tokenSummarySource"] == "final", cell
     assert cell["toolSummary"]["actualToolCallCounts"] == {"command_execution": 1}, cell
     assert "ephemeral-token" not in json.dumps(cell), cell
 
@@ -4934,6 +4941,10 @@ with tempfile.TemporaryDirectory() as tmp:
             "MAKA_PROVIDER_PROXY_TOKEN": "ephemeral-token",
             "MAKA_MODEL": "gpt-5.6-sol",
             "MAKA_SYSTEM_PROMPT": "",
+            "MAKA_TRIAL_INPUT_USD_PER_1M": "5",
+            "MAKA_TRIAL_CACHE_READ_USD_PER_1M": "0.5",
+            "MAKA_TRIAL_OUTPUT_USD_PER_1M": "30",
+            "MAKA_TRIAL_PRICING_SOURCE": "openai-gpt-5.6-sol-2026-07-20",
         },
     )
     try:
@@ -4946,6 +4957,7 @@ with tempfile.TemporaryDirectory() as tmp:
     failed = json.loads((logs / "maka-cell-output.json").read_text(encoding="utf-8"))
     assert failed["status"] == "failed", failed
     assert failed["errorClass"] == "auth", failed
+    assert failed["tokenSummarySource"] == "checkpoint", failed
 
     transport = MakaCodexAgent(
         logs,

packages/headless/src/__tests__/harbor-cell.test.ts

@@ -901,10 +901,8 @@ describe('runHarborCell', () => {
         costUsd: 0.012,
         pricingSource: 'runtime',
       });
-      assert.deepEqual(
-        JSON.parse(await readFile(result.outputPath, 'utf8')).tokenSummary,
-        result.output.tokenSummary,
-      );
+      assert.equal(result.output.tokenSummarySource, 'checkpoint');
+      assert.deepEqual(JSON.parse(await readFile(result.outputPath, 'utf8')), result.output);
     });
   });