From 93272f04d42c246b6baec8df9e140971a3c42a2e Mon Sep 17 00:00:00 2001 From: likun666661 <90952590+likun666661@users.noreply.github.com> Date: Tue, 23 Jun 2026 15:08:10 +0800 Subject: [PATCH] Add heavy-task dual completion status --- .../__tests__/heavy-task-finalization.test.ts | 202 +++++++++++++++++ .../src/__tests__/result-export.test.ts | 157 ++++++++++++- .../headless/src/__tests__/scorer.test.ts | 28 +++ .../src/__tests__/task-run-store.test.ts | 76 +++++++ .../headless/src/heavy-task-finalization.ts | 211 ++++++++++++++++++ packages/headless/src/index.ts | 9 + packages/headless/src/result-export.ts | 9 + packages/headless/src/task-run-store.ts | 20 ++ 8 files changed, 711 insertions(+), 1 deletion(-) create mode 100644 packages/headless/src/__tests__/heavy-task-finalization.test.ts create mode 100644 packages/headless/src/heavy-task-finalization.ts diff --git a/packages/headless/src/__tests__/heavy-task-finalization.test.ts b/packages/headless/src/__tests__/heavy-task-finalization.test.ts new file mode 100644 index 0000000000..09c928e9d8 --- /dev/null +++ b/packages/headless/src/__tests__/heavy-task-finalization.test.ts @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import { evaluateHeavyTaskCompletionStatus } from '../heavy-task-finalization.js'; +import type { + HeavyTaskModeFacts, + HeavyTaskSemanticSelfCheckState, + HeavyTaskSelfCheckStatus, + HeavyTaskTodoItem, + HeavyTaskTodoState, + TaskRunStatus, +} from '../task-contracts.js'; + +const heavyTaskMode: HeavyTaskModeFacts = { + schemaVersion: 1, + enabled: true, + triggerSource: 'config', + triggerReason: 'long public task', + policyVersion: 'maka-heavy-task-policy.v1', +}; + +describe('heavy-task finalization status', () => { + test('marks semantic complete with accepted pass self-check and completed todos', () => { + const status = evaluateHeavyTaskCompletionStatus({ + status: 'budget_exhausted', + taxonomy: 'budget_exhausted', + heavyTaskMode, + latestHeavyTaskSelfCheck: selfCheck('pass'), + latestHeavyTaskTodos: todos([{ id: 'edit', status: 'completed' }]), + }); + + assert.equal(status.runtime.capLike, true); + assert.equal(status.runtime.capKind, 'budget_exhausted'); + assert.equal(status.semantic.status, 'complete'); + assert.equal(status.semantic.advisory, true); + assert.deepEqual(status.semantic.unresolvedTodoIds, []); + assert.equal(status.finalization.eligible, true); + assert.equal(status.finalization.boundedTurnImplemented, false); + }); + + test('treats cancelled todos with evidence as nonblocking', () => { + const status = evaluateHeavyTaskCompletionStatus({ + status: 'incomplete', + taxonomy: 'agent_incomplete', + heavyTaskMode, + latestHeavyTaskSelfCheck: selfCheck('pass'), + latestHeavyTaskTodos: todos([ + { id: 'implemented', status: 'completed' }, + { id: 'optional-polish', status: 'cancelled', evidence: 'Out of scope after public README review.' }, + ]), + }); + + assert.equal(status.semantic.status, 'complete'); + assert.deepEqual(status.semantic.nonblockingTodoIds, ['optional-polish']); + assert.deepEqual(status.semantic.unresolvedTodoIds, []); + assert.equal(status.finalization.eligible, true); + }); + + test('requires accepted public pass self-check evidence', () => { + const cases = [ + { name: 'missing self-check', selfCheck: undefined }, + { + name: 'rejected self-check', + selfCheck: selfCheck('pass', { guardStatus: 'rejected' }) as unknown as HeavyTaskSemanticSelfCheckState, + }, + { name: 'private payload replay', selfCheck: selfCheck('pass', { publicReason: 'hidden/tests/private_case.py passed.' }) }, + { name: 'failed self-check', selfCheck: selfCheck('fail') }, + { name: 'inconclusive self-check', selfCheck: selfCheck('inconclusive') }, + ]; + + for (const item of cases) { + const status = evaluateHeavyTaskCompletionStatus({ + status: 'budget_exhausted', + taxonomy: 'budget_exhausted', + heavyTaskMode, + latestHeavyTaskSelfCheck: item.selfCheck, + latestHeavyTaskTodos: todos([{ id: 'edit', status: 'completed' }]), + }); + + assert.equal(status.semantic.status, 'incomplete', item.name); + assert.equal(status.finalization.eligible, false, item.name); + } + }); + + test('requires non-empty latest todos with no unresolved work', () => { + const cases = [ + { name: 'missing todos', todos: undefined, unresolved: [] }, + { name: 'empty todos', todos: todos([]), unresolved: [] }, + { name: 'pending todo', todos: todos([{ id: 'inspect', status: 'pending' }]), unresolved: ['inspect'] }, + { name: 'in-progress todo', todos: todos([{ id: 'edit', status: 'in_progress' }]), unresolved: ['edit'] }, + { name: 'cancelled without evidence', todos: todos([{ id: 'optional', status: 'cancelled' }]), unresolved: ['optional'] }, + { + name: 'unknown future status', + todos: todos([{ id: 'future', status: 'blocked' as HeavyTaskTodoItem['status'] }]), + unresolved: ['future'], + }, + ]; + + for (const item of cases) { + const status = evaluateHeavyTaskCompletionStatus({ + status: 'budget_exhausted', + taxonomy: 'budget_exhausted', + heavyTaskMode, + latestHeavyTaskSelfCheck: selfCheck('pass'), + latestHeavyTaskTodos: item.todos, + }); + + assert.equal(status.semantic.status, 'incomplete', item.name); + assert.deepEqual(status.semantic.unresolvedTodoIds, item.unresolved, item.name); + assert.equal(status.finalization.eligible, false, item.name); + } + }); + + test('classifies cap-like runtime outcomes without treating verifier failures as caps', () => { + const capCases: Array<{ + name: string; + status: TaskRunStatus; + taxonomy?: string; + errorClass?: string; + message?: string; + reason?: string; + capKind: string; + }> = [ + { name: 'budget exhausted', status: 'budget_exhausted', taxonomy: 'budget_exhausted', capKind: 'budget_exhausted' }, + { name: 'runtime step cap', status: 'failed', errorClass: 'max_steps', message: 'runtime step cap reached', capKind: 'runtime_step_cap' }, + { name: 'wall time cap', status: 'failed', message: 'wall time cap reached', capKind: 'wall_time_cap' }, + { name: 'max attempts', status: 'failed', reason: 'max attempts exhausted', capKind: 'max_attempts' }, + { name: 'tool calls', status: 'incomplete', errorClass: 'incomplete_tool_calls', capKind: 'tool_call_step_cap' }, + { name: 'max tokens', status: 'incomplete', errorClass: 'max_tokens', capKind: 'token_cap' }, + { name: 'timeout', status: 'failed', errorClass: 'timeout', capKind: 'timeout' }, + ]; + + for (const item of capCases) { + const status = evaluateHeavyTaskCompletionStatus({ + status: item.status, + taxonomy: item.taxonomy, + error: item.errorClass || item.message ? { class: item.errorClass, message: item.message ?? item.errorClass ?? '' } : undefined, + decisions: item.reason ? [{ id: `decision-${item.name}`, taskRunId: 'run-1', ts: 1, decision: 'stop', reason: item.reason }] : undefined, + heavyTaskMode, + latestHeavyTaskSelfCheck: selfCheck('pass'), + latestHeavyTaskTodos: todos([{ id: 'edit', status: 'completed' }]), + }); + + assert.equal(status.runtime.capLike, true, item.name); + assert.equal(status.runtime.capKind, item.capKind, item.name); + assert.equal(status.finalization.eligible, true, item.name); + } + + const verifierFailure = evaluateHeavyTaskCompletionStatus({ + status: 'completed', + taxonomy: 'verification_failed', + heavyTaskMode, + latestHeavyTaskSelfCheck: selfCheck('pass'), + latestHeavyTaskTodos: todos([{ id: 'edit', status: 'completed' }]), + }); + assert.equal(verifierFailure.runtime.capLike, false); + assert.equal(verifierFailure.runtime.capKind, 'none'); + assert.equal(verifierFailure.semantic.status, 'complete'); + assert.equal(verifierFailure.finalization.eligible, false); + }); +}); + +function selfCheck( + status: HeavyTaskSelfCheckStatus, + options: { guardStatus?: 'accepted' | 'rejected'; publicReason?: string } = {}, +): HeavyTaskSemanticSelfCheckState { + return { + schemaVersion: 1, + selfCheckId: `self-check-${status}-${options.guardStatus ?? 'accepted'}`, + taskRunId: 'run-1', + ts: 2, + status, + publicReason: options.publicReason ?? 'npm test passed against public files.', + commandEvidence: [{ command: 'npm test', exitCode: 0, outputExcerpt: 'public tests passed' }], + artifactEvidence: [{ path: 'build-output.log', kind: 'log', exists: true }], + guard: { + status: options.guardStatus ?? 'accepted', + checkedAt: 2, + categories: options.guardStatus === 'rejected' ? ['official_verifier_artifacts'] : [], + publicReason: options.guardStatus === 'rejected' + ? 'Rejected because submitted evidence referenced private, hidden, or evaluator-only material.' + : 'Accepted as public, task-derived advisory self-check evidence.', + } as unknown as HeavyTaskSemanticSelfCheckState['guard'], + source: { kind: 'model_tool', toolCallId: 'tool-self-check' }, + }; +} + +function todos(items: Array<{ id: string; status: HeavyTaskTodoItem['status']; evidence?: string }>): HeavyTaskTodoState { + return { + schemaVersion: 1, + todoSetId: 'todos-1', + taskRunId: 'run-1', + ts: 3, + items: items.map((item) => ({ + id: item.id, + content: `Work item ${item.id}`, + status: item.status, + priority: 'high', + ...(item.evidence ? { evidence: item.evidence } : {}), + })), + source: { kind: 'model_tool', toolCallId: 'tool-todos' }, + }; +} diff --git a/packages/headless/src/__tests__/result-export.test.ts b/packages/headless/src/__tests__/result-export.test.ts index b0e459632e..6892436c66 100644 --- a/packages/headless/src/__tests__/result-export.test.ts +++ b/packages/headless/src/__tests__/result-export.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; import { renderTaskRunMarkdown, taskRunExportFromProjection, writeTaskRunExport } from '../result-export.js'; -import type { TaskEvent } from '../task-contracts.js'; +import type { HeavyTaskTodoItem, TaskEvent } from '../task-contracts.js'; import { projectTaskRun } from '../task-run-store.js'; describe('task run export', () => { @@ -283,6 +283,55 @@ describe('task run export', () => { } }); + test('exports heavy-task dual runtime and semantic completion status in full and compact views', async () => { + const projection = projectTaskRun(heavyTaskCompletionEvents(), 'run-heavy-complete'); + const exported = taskRunExportFromProjection(projection, { exportedAt: '2026-06-19T00:00:00.000Z' }); + + assert.equal(exported.taskRun.status, 'budget_exhausted'); + assert.equal(exported.taxonomy.value, 'verification_failed'); + assert.equal(exported.taxonomy.passed, false); + assert.equal(exported.legacyResultRecord.passed, false); + assert.equal(exported.heavyTask?.mode?.enabled, true); + assert.equal(exported.heavyTask?.completion.runtime.taskRunStatus, 'budget_exhausted'); + assert.equal(exported.heavyTask?.completion.runtime.taxonomy, 'verification_failed'); + assert.equal(exported.heavyTask?.completion.runtime.capLike, true); + assert.equal(exported.heavyTask?.completion.semantic.status, 'complete'); + assert.equal(exported.heavyTask?.completion.semantic.advisory, true); + assert.deepEqual(exported.heavyTask?.completion.semantic.unresolvedTodoIds, []); + assert.deepEqual(exported.heavyTask?.completion.semantic.nonblockingTodoIds, ['optional-polish']); + assert.equal(exported.heavyTask?.completion.finalization.eligible, true); + assert.equal(exported.score?.taxonomy, 'verification_failed'); + assert.equal(exported.verifier?.passed, false); + + const outDir = await mkdtemp(join(tmpdir(), 'maka-heavy-task-export-')); + try { + const written = await writeTaskRunExport(outDir, projection, { + exportedAt: '2026-06-19T00:00:00.000Z', + }); + const compact = JSON.parse(await readFile(written.files.resultJson, 'utf8')); + assert.deepEqual(compact.heavyTask, exported.heavyTask); + assert.equal(compact.taxonomy.value, 'verification_failed'); + assert.equal(compact.legacyResultRecord.passed, false); + } finally { + await rm(outDir, { recursive: true, force: true }); + } + }); + + test('exports semantic incomplete and finalization ineligible when todos are unresolved', () => { + const events = heavyTaskCompletionEvents([ + { id: 'edit', content: 'Patch implementation', status: 'completed', priority: 'high' }, + { id: 'verify', content: 'Run public checks', status: 'pending', priority: 'high' }, + ]); + const exported = taskRunExportFromProjection(projectTaskRun(events, 'run-heavy-complete')); + + assert.equal(exported.heavyTask?.completion.runtime.capLike, true); + assert.equal(exported.heavyTask?.completion.semantic.status, 'incomplete'); + assert.deepEqual(exported.heavyTask?.completion.semantic.unresolvedTodoIds, ['verify']); + assert.equal(exported.heavyTask?.completion.finalization.eligible, false); + assert.equal(exported.taxonomy.value, 'verification_failed'); + assert.equal(exported.legacyResultRecord.passed, false); + }); + test('exports official verifier truth over a non-authoritative placeholder result', async () => { const events: TaskEvent[] = [ { type: 'task_run_created', id: 'e1', taskRunId: 'run-official', ts: 1, taskId: 'task-1', configId: 'cfg-1' }, @@ -551,3 +600,109 @@ describe('task run export', () => { } }); }); + +function heavyTaskCompletionEvents(todos: HeavyTaskTodoItem[] = [ + { id: 'edit', content: 'Patch implementation', status: 'completed', priority: 'high' }, + { + id: 'optional-polish', + content: 'Optional polish', + status: 'cancelled', + priority: 'low', + evidence: 'Not required by public task.', + }, +]): TaskEvent[] { + const taskRunId = 'run-heavy-complete'; + return [ + { type: 'task_run_created', id: 'hc1', taskRunId, ts: 1, taskId: 'task-1', configId: 'cfg-1' }, + { + type: 'heavy_task_mode_recorded', + id: 'hc2', + taskRunId, + ts: 2, + facts: { + schemaVersion: 1, + enabled: true, + triggerSource: 'config', + triggerReason: 'long public task', + policyVersion: 'maka-heavy-task-policy.v1', + }, + }, + { + type: 'heavy_task_todos_recorded', + id: 'hc3', + taskRunId, + ts: 3, + todos: { + schemaVersion: 1, + todoSetId: 'todos-2', + taskRunId, + ts: 3, + items: todos, + source: { kind: 'model_tool', toolCallId: 'tool-todos' }, + }, + }, + { + type: 'heavy_task_self_check_recorded', + id: 'hc4', + taskRunId, + ts: 4, + selfCheck: { + schemaVersion: 1, + selfCheckId: 'self-check-1', + taskRunId, + ts: 4, + status: 'pass', + publicReason: 'npm test passed against public files.', + commandEvidence: [{ command: 'npm test', exitCode: 0, outputExcerpt: 'public tests passed' }], + artifactEvidence: [{ path: 'build-output.log', kind: 'log', exists: true }], + guard: { + status: 'accepted', + checkedAt: 4, + categories: [], + publicReason: 'Accepted as public, task-derived advisory self-check evidence.', + }, + source: { kind: 'model_tool', toolCallId: 'tool-self-check' }, + }, + }, + { + type: 'verifier_result_recorded', + id: 'hc5', + taskRunId, + ts: 5, + result: { + id: 'verifier-1', + taskRunId, + ts: 5, + kind: 'terminal_bench', + passed: false, + exitCode: 1, + errorClass: 'verification_failed', + authority: { source: 'official_harbor_verifier', authoritative: true }, + }, + }, + { + type: 'score_result_recorded', + id: 'hc6', + taskRunId, + ts: 6, + result: { + id: 'score-1', + taskRunId, + ts: 6, + passed: false, + scored: true, + eligible: true, + taxonomy: 'verification_failed', + errorClass: 'verification_failed', + authority: { source: 'official_harbor_verifier', authoritative: true }, + }, + }, + { + type: 'task_run_budget_exhausted', + id: 'hc7', + taskRunId, + ts: 7, + error: { message: 'runtime step cap reached', class: 'max_steps' }, + }, + ]; +} diff --git a/packages/headless/src/__tests__/scorer.test.ts b/packages/headless/src/__tests__/scorer.test.ts index 1362e8bdec..2560c67f02 100644 --- a/packages/headless/src/__tests__/scorer.test.ts +++ b/packages/headless/src/__tests__/scorer.test.ts @@ -41,4 +41,32 @@ describe('defaultFinalScorer', () => { assert.equal(score.taxonomy, 'agent_incomplete'); assert.equal(score.errorClass, 'incomplete_tool_calls'); }); + + test('keeps official verifier failure authoritative over advisory semantic completion', () => { + const verifierResult: VerifierResult = { + id: 'verifier', + taskRunId: 'run', + ts: 1, + kind: 'terminal_bench', + passed: false, + exitCode: 1, + errorClass: 'verification_failed', + authority: { source: 'official_harbor_verifier', authoritative: true }, + }; + + const score = defaultFinalScorer({ + config, + task, + runnerCompleted: true, + runnerStatus: 'completed', + submittedSnapshot, + verifierResult, + }); + + assert.equal(score.passed, false); + assert.equal(score.scored, true); + assert.equal(score.eligible, true); + assert.equal(score.taxonomy, 'verification_failed'); + assert.equal(score.errorClass, 'verification_failed'); + }); }); diff --git a/packages/headless/src/__tests__/task-run-store.test.ts b/packages/headless/src/__tests__/task-run-store.test.ts index a129f99884..6748f522cd 100644 --- a/packages/headless/src/__tests__/task-run-store.test.ts +++ b/packages/headless/src/__tests__/task-run-store.test.ts @@ -240,6 +240,82 @@ describe('TaskRunStore', () => { assert.match(projection.warnings.join('\n'), /source guard did not accept/); }); + test('derives heavy-task completion from accepted self-checks and latest todos', () => { + const taskRunId = 'tr-heavy-completion'; + const projection = projectTaskRun([ + { type: 'task_run_created', id: 'e-1', taskRunId, ts: 1, taskId: 'task-1', configId: 'cfg-1' }, + { + type: 'heavy_task_mode_recorded', + id: 'e-2', + taskRunId, + ts: 2, + facts: { + schemaVersion: 1, + enabled: true, + triggerSource: 'config', + triggerReason: 'long public task', + policyVersion: 'maka-heavy-task-policy.v1', + }, + }, + { + type: 'heavy_task_todos_recorded', + id: 'e-3', + taskRunId, + ts: 3, + todos: { + schemaVersion: 1, + todoSetId: 'todos-1', + taskRunId, + ts: 3, + items: [ + { id: 'edit', content: 'Patch implementation', status: 'completed', priority: 'high' }, + { id: 'optional', content: 'Optional polish', status: 'cancelled', priority: 'low', evidence: 'Not required by public task.' }, + ], + source: { kind: 'model_tool', toolCallId: 'tool-2' }, + }, + }, + { + type: 'heavy_task_self_check_recorded', + id: 'e-4', + taskRunId, + ts: 4, + selfCheck: acceptedSelfCheck(taskRunId, 'self-check-1', 'pass', 'npm test passed against public files.'), + }, + { + type: 'score_result_recorded', + id: 'e-5', + taskRunId, + ts: 5, + result: { + id: 'score-1', + taskRunId, + ts: 5, + passed: false, + scored: true, + eligible: true, + taxonomy: 'verification_failed', + authority: { source: 'official_harbor_verifier', authoritative: true }, + }, + }, + { + type: 'task_run_budget_exhausted', + id: 'e-6', + taskRunId, + ts: 6, + error: { message: 'runtime step cap reached', class: 'max_steps' }, + }, + ], taskRunId); + + assert.equal(projection.heavyTaskCompletion?.runtime.taskRunStatus, 'budget_exhausted'); + assert.equal(projection.heavyTaskCompletion?.runtime.taxonomy, 'verification_failed'); + assert.equal(projection.heavyTaskCompletion?.runtime.capKind, 'runtime_step_cap'); + assert.equal(projection.heavyTaskCompletion?.semantic.status, 'complete'); + assert.deepEqual(projection.heavyTaskCompletion?.semantic.nonblockingTodoIds, ['optional']); + assert.equal(projection.heavyTaskCompletion?.finalization.eligible, true); + assert.equal(projection.result?.taxonomy, 'verification_failed'); + assert.equal(projection.result?.passed, false); + }); + test('projects isolation, permission, inbox, and needs_approval facts', () => { const taskRunId = 'tr-approval'; const request = { diff --git a/packages/headless/src/heavy-task-finalization.ts b/packages/headless/src/heavy-task-finalization.ts new file mode 100644 index 0000000000..ed78bd7f89 --- /dev/null +++ b/packages/headless/src/heavy-task-finalization.ts @@ -0,0 +1,211 @@ +import { isAcceptedHeavyTaskSelfCheck } from './heavy-task-self-check.js'; +import type { + AutonomousDecision, + AutonomousResultTaxonomy, + HeavyTaskModeFacts, + HeavyTaskSelfCheckStatus, + HeavyTaskSemanticSelfCheckState, + HeavyTaskTodoItem, + HeavyTaskTodoState, + TaskRunError, + TaskRunStatus, +} from './task-contracts.js'; + +export type HeavyTaskRuntimeCapKind = + | 'none' + | 'tool_call_step_cap' + | 'token_cap' + | 'runtime_step_cap' + | 'wall_time_cap' + | 'max_attempts' + | 'timeout' + | 'budget_exhausted' + | 'unknown_cap'; + +export type HeavyTaskSemanticStatus = 'complete' | 'incomplete'; + +export interface HeavyTaskCompletionStatus { + schemaVersion: 1; + runtime: { + taskRunStatus: TaskRunStatus; + taxonomy?: AutonomousResultTaxonomy | string; + capLike: boolean; + capKind: HeavyTaskRuntimeCapKind; + failureClass?: string; + reason?: string; + }; + semantic: { + status: HeavyTaskSemanticStatus; + advisory: true; + reason: string; + selfCheckId?: string; + selfCheckStatus?: HeavyTaskSelfCheckStatus; + todoSetId?: string; + unresolvedTodoIds: string[]; + nonblockingTodoIds: string[]; + }; + finalization: { + eligible: boolean; + reason: string; + boundedTurnImplemented: false; + }; +} + +export interface HeavyTaskCompletionInput { + status: TaskRunStatus; + taxonomy?: AutonomousResultTaxonomy | string; + error?: TaskRunError; + heavyTaskMode?: HeavyTaskModeFacts; + latestHeavyTaskTodos?: HeavyTaskTodoState; + latestHeavyTaskSelfCheck?: HeavyTaskSemanticSelfCheckState; + decisions?: readonly AutonomousDecision[]; +} + +export function evaluateHeavyTaskCompletionStatus(input: HeavyTaskCompletionInput): HeavyTaskCompletionStatus { + const runtime = runtimeStatusFromInput(input); + const semantic = semanticStatusFromInput(input); + const eligible = semantic.status === 'complete' && runtime.capLike; + return { + schemaVersion: 1, + runtime, + semantic, + finalization: { + eligible, + reason: eligible + ? 'runtime cap outcome with accepted semantic completion evidence' + : finalizationIneligibleReason(runtime, semantic), + boundedTurnImplemented: false, + }, + }; +} + +function runtimeStatusFromInput(input: HeavyTaskCompletionInput): HeavyTaskCompletionStatus['runtime'] { + const failureClass = input.error?.class; + const reason = runtimeReason(input); + const capKind = classifyCapKind(input, reason); + return { + taskRunStatus: input.status, + ...(input.taxonomy ? { taxonomy: input.taxonomy } : {}), + capLike: capKind !== 'none', + capKind, + ...(failureClass ? { failureClass } : {}), + ...(reason ? { reason } : {}), + }; +} + +function semanticStatusFromInput(input: HeavyTaskCompletionInput): HeavyTaskCompletionStatus['semantic'] { + const selfCheck = input.latestHeavyTaskSelfCheck; + const todos = input.latestHeavyTaskTodos; + const unresolvedTodoIds = unresolvedTodoIdsFrom(todos); + const nonblockingTodoIds = nonblockingTodoIdsFrom(todos); + const base = { + advisory: true as const, + ...(selfCheck ? { selfCheckId: selfCheck.selfCheckId, selfCheckStatus: selfCheck.status } : {}), + ...(todos ? { todoSetId: todos.todoSetId } : {}), + unresolvedTodoIds, + nonblockingTodoIds, + }; + + if (input.heavyTaskMode?.enabled !== true) { + return { ...base, status: 'incomplete', reason: 'heavy-task mode is not enabled' }; + } + if (!selfCheck) { + return { ...base, status: 'incomplete', reason: 'missing accepted public self-check evidence' }; + } + if (!isAcceptedHeavyTaskSelfCheck(selfCheck)) { + return { ...base, status: 'incomplete', reason: 'latest self-check evidence was not accepted as public' }; + } + if (selfCheck.status !== 'pass') { + return { ...base, status: 'incomplete', reason: `latest self-check status is ${selfCheck.status}` }; + } + if (!todos) { + return { ...base, status: 'incomplete', reason: 'missing latest heavy-task todos' }; + } + if (todos.items.length === 0) { + return { ...base, status: 'incomplete', reason: 'latest heavy-task todos are empty' }; + } + if (unresolvedTodoIds.length > 0) { + return { ...base, status: 'incomplete', reason: 'latest heavy-task todos contain unresolved work' }; + } + return { + ...base, + status: 'complete', + reason: 'accepted public self-check passed and latest todos are resolved/nonblocking', + }; +} + +function unresolvedTodoIdsFrom(todos: HeavyTaskTodoState | undefined): string[] { + if (!todos) return []; + return todos.items.filter((item) => !isResolvedOrNonblockingTodo(item)).map((item) => item.id); +} + +function nonblockingTodoIdsFrom(todos: HeavyTaskTodoState | undefined): string[] { + if (!todos) return []; + return todos.items.filter(isNonblockingTodo).map((item) => item.id); +} + +function isResolvedOrNonblockingTodo(item: HeavyTaskTodoItem): boolean { + return item.status === 'completed' || isNonblockingTodo(item); +} + +function isNonblockingTodo(item: HeavyTaskTodoItem): boolean { + return item.status === 'cancelled' && typeof item.evidence === 'string' && item.evidence.trim().length > 0; +} + +function classifyCapKind(input: HeavyTaskCompletionInput, reason: string | undefined): HeavyTaskRuntimeCapKind { + const haystack = [ + input.status, + input.taxonomy, + input.error?.class, + input.error?.message, + reason, + ...decisionReasons(input.decisions), + ].filter((value): value is string => typeof value === 'string' && value.length > 0).join(' ').toLowerCase(); + + if (haystack.includes('incomplete_tool_calls') || haystack.includes('tool_call_step') || haystack.includes('tool call step')) { + return 'tool_call_step_cap'; + } + if (haystack.includes('max_tokens') || haystack.includes('max token') || haystack.includes('token cap') || haystack.includes('truncated')) { + return 'token_cap'; + } + if (haystack.includes('runtime step cap') || haystack.includes('max_steps') || haystack.includes('max steps')) { + return 'runtime_step_cap'; + } + if (haystack.includes('wall time cap') || haystack.includes('wall-time cap') || haystack.includes('wall_time')) { + return 'wall_time_cap'; + } + if (haystack.includes('max attempts') || haystack.includes('max_attempts')) { + return 'max_attempts'; + } + if (haystack.includes('timeout') || haystack.includes('timed out') || haystack.includes('timed_out')) { + return 'timeout'; + } + if (input.status === 'budget_exhausted' || input.taxonomy === 'budget_exhausted' || haystack.includes('budget exhausted')) { + return 'budget_exhausted'; + } + if (input.status === 'incomplete' || input.taxonomy === 'agent_incomplete') { + return 'unknown_cap'; + } + if (haystack.includes('tool_calls') || haystack.includes('tool calls') || haystack.includes('budget') || haystack.includes('limit')) { + return 'unknown_cap'; + } + return 'none'; +} + +function runtimeReason(input: HeavyTaskCompletionInput): string | undefined { + const decisionReason = [...(input.decisions ?? [])].reverse().find((decision) => decision.reason)?.reason; + return input.error?.message ?? decisionReason; +} + +function decisionReasons(decisions: readonly AutonomousDecision[] | undefined): string[] { + return (decisions ?? []).map((decision) => decision.reason).filter((reason): reason is string => typeof reason === 'string'); +} + +function finalizationIneligibleReason( + runtime: HeavyTaskCompletionStatus['runtime'], + semantic: HeavyTaskCompletionStatus['semantic'], +): string { + if (semantic.status !== 'complete') return 'semantic completion evidence is incomplete'; + if (!runtime.capLike) return 'runtime outcome is not cap-like'; + return 'finalization is not eligible'; +} diff --git a/packages/headless/src/index.ts b/packages/headless/src/index.ts index 580b28fed9..1ffaf8ba08 100644 --- a/packages/headless/src/index.ts +++ b/packages/headless/src/index.ts @@ -328,6 +328,15 @@ export { renderHeavyTaskSelfCheckForPrompt, validateHeavyTaskPublicSelfCheck, } from './heavy-task-self-check.js'; +export type { + HeavyTaskCompletionInput, + HeavyTaskCompletionStatus, + HeavyTaskRuntimeCapKind, + HeavyTaskSemanticStatus, +} from './heavy-task-finalization.js'; +export { + evaluateHeavyTaskCompletionStatus, +} from './heavy-task-finalization.js'; export type { HeadlessBackendContext, IsolatedCommandInput, diff --git a/packages/headless/src/result-export.ts b/packages/headless/src/result-export.ts index c4cd445018..8e3cd50ba9 100644 --- a/packages/headless/src/result-export.ts +++ b/packages/headless/src/result-export.ts @@ -54,6 +54,10 @@ export interface TaskRunExport { policy?: { heavyTask?: TaskRunProjection['heavyTaskMode']; }; + heavyTask?: { + mode?: TaskRunProjection['heavyTaskMode']; + completion: NonNullable; + }; progress?: { inventory?: { latest: NonNullable; @@ -143,6 +147,9 @@ export function taskRunExportFromProjection( const taxonomy = score?.taxonomy ?? projection.result?.taxonomy ?? legacyResultRecord.errorClass ?? projection.status; const primaryWorkspacePath = primaryWorkspacePathFromArtifacts(projection.artifacts); const policy = projection.heavyTaskMode?.enabled ? { heavyTask: projection.heavyTaskMode } : undefined; + const heavyTask = projection.heavyTaskCompletion + ? { mode: projection.heavyTaskMode, completion: projection.heavyTaskCompletion } + : undefined; const progress = progressFromProjection(projection); return { @@ -189,6 +196,7 @@ export function taskRunExportFromProjection( score, budget: recordValue(scoreDetails.budget) ? scoreDetails.budget as Record : undefined, policy, + heavyTask, progress, isolation: { policy: projection.isolation, @@ -275,6 +283,7 @@ function compactResultView(exported: TaskRunExport): Record { : undefined, score: exported.score, policy: exported.policy, + heavyTask: exported.heavyTask, progress: exported.progress, workspace: exported.workspace, artifacts: exported.artifacts, diff --git a/packages/headless/src/task-run-store.ts b/packages/headless/src/task-run-store.ts index fa3db3fee9..5d9caa5f2e 100644 --- a/packages/headless/src/task-run-store.ts +++ b/packages/headless/src/task-run-store.ts @@ -1,6 +1,7 @@ import { appendFile, mkdir, readFile } from 'node:fs/promises'; import { join } from 'node:path'; import type { ResultRecord } from './contracts.js'; +import { evaluateHeavyTaskCompletionStatus, type HeavyTaskCompletionStatus } from './heavy-task-finalization.js'; import { isAcceptedHeavyTaskSelfCheck } from './heavy-task-self-check.js'; import type { AutonomousDecision, @@ -50,6 +51,7 @@ export interface TaskRunProjection extends TaskRun { latestHeavyTaskTodos?: HeavyTaskTodoState; heavyTaskSelfChecks: HeavyTaskSemanticSelfCheckState[]; latestHeavyTaskSelfCheck?: HeavyTaskSemanticSelfCheckState; + heavyTaskCompletion?: HeavyTaskCompletionStatus; isolation?: TaskIsolationFacts; workspaceLease?: WorkspaceLeaseFacts; parked?: TaskRunParkedState; @@ -301,6 +303,17 @@ export function projectTaskRun(events: readonly TaskEvent[], taskRunId?: string) } else if (projection.latestVerifierResult) { projection.result = resultFromVerifier(projection.latestVerifierResult); } + if (hasHeavyTaskCompletionState(projection)) { + projection.heavyTaskCompletion = evaluateHeavyTaskCompletionStatus({ + status: projection.status, + taxonomy: projection.latestScoreResult?.taxonomy ?? projection.result?.taxonomy, + error: projection.error, + heavyTaskMode: projection.heavyTaskMode, + latestHeavyTaskTodos: projection.latestHeavyTaskTodos, + latestHeavyTaskSelfCheck: projection.latestHeavyTaskSelfCheck, + decisions: projection.decisions, + }); + } return projection; } @@ -454,6 +467,13 @@ function applyTerminalEvent(projection: TaskRunProjection, terminalEvents: numbe return terminalEvents + 1; } +function hasHeavyTaskCompletionState(projection: TaskRunProjection): boolean { + return projection.heavyTaskMode?.enabled === true + || projection.heavyTaskInventory.length > 0 + || projection.heavyTaskTodoStates.length > 0 + || projection.heavyTaskSelfChecks.length > 0; +} + function safeFileId(id: string): string { return id.replace(/[^A-Za-z0-9._-]/g, '_') || '_'; }