From 1d378a82f50f6bd3a0ebce646e835fdaa47bf92e Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 17:48:51 +0800 Subject: [PATCH 1/8] feat(headless): consolidate Terminal-Bench smoke entry into packages/headless Add the pure-Node smoke orchestrator run-terminal-bench-smoke.mjs plus a checked-in profile manifest and a testable job-config generator (harbor-smoke-config.ts, exported from src/index.ts). Maka profiles target the authoritative maka_agent:MakaAgent adapter in task-run host-bridge mode (MAKA_HARBOR_MODE=task-run); relocate the OpenCode title wrapper into the harbor adapter dir. Generated smoke jobs/configs are gitignored. --- packages/headless/.gitignore | 2 + packages/headless/README.md | 21 ++ .../harbor}/opencode_title_harbor_agent.py | 0 .../harbor/run-terminal-bench-smoke.mjs | 190 +++++++++++++ .../harbor/terminal-bench-smoke-profiles.json | 155 +++++++++++ packages/headless/package.json | 1 + .../src/__tests__/harbor-smoke-config.test.ts | 162 +++++++++++ packages/headless/src/harbor-smoke-config.ts | 263 ++++++++++++++++++ packages/headless/src/index.ts | 14 + 9 files changed, 808 insertions(+) create mode 100644 packages/headless/.gitignore rename {terminal-bench-smoke => packages/headless/harbor}/opencode_title_harbor_agent.py (100%) create mode 100644 packages/headless/harbor/run-terminal-bench-smoke.mjs create mode 100644 packages/headless/harbor/terminal-bench-smoke-profiles.json create mode 100644 packages/headless/src/__tests__/harbor-smoke-config.test.ts create mode 100644 packages/headless/src/harbor-smoke-config.ts diff --git a/packages/headless/.gitignore b/packages/headless/.gitignore new file mode 100644 index 0000000000..c649c8c637 --- /dev/null +++ b/packages/headless/.gitignore @@ -0,0 +1,2 @@ +/harbor/smoke-jobs/ +/harbor/smoke-generated-configs/ diff --git a/packages/headless/README.md b/packages/headless/README.md index 47ebea15d3..24462f6b2e 100644 --- a/packages/headless/README.md +++ b/packages/headless/README.md @@ -161,6 +161,27 @@ trajectory/runtime refs, submitted snapshot metadata, verifier output, score, budget, isolation, permission/inbox facts, taxonomy, and warnings. They do not embed environment variables, credentials, or hidden harness configuration. +## Terminal-Bench smoke runner + +`harbor/run-terminal-bench-smoke.mjs` is the local structured smoke harness for the +`terminal-bench-sample` registry dataset. It reads the checked-in profile manifest +`harbor/terminal-bench-smoke-profiles.json`, generates a Harbor run config under +`harbor/smoke-generated-configs/`, and (unless `--dry-run`) invokes Harbor with the +adapter directory on `PYTHONPATH`. `HARBOR_BIN` overrides the Harbor executable +(default `harbor` on `PATH`). + +The `maka-*` profiles drive the single authoritative adapter `maka_agent:MakaAgent` +in task-run host-bridge mode (`MAKA_HARBOR_MODE=task-run`): Maka runs the full +task-run controller on the host and bridges tool execution into the task container, +while the container installs nothing. `maka-heavy` and `maka-heavy-prune` carry the +heavy-task and autonomous prior-attempt-replay experiments; `opencode` and `oracle` +provide comparison and cheap dataset smoke arms. + +```sh +node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-heavy --dry-run +node packages/headless/harbor/run-terminal-bench-smoke.mjs --compare --task '*sqlite-with-gcov' +``` + ## GLM-5.2 harness comparison `harbor/run-harness-ab.mjs` compares Maka and OpenCode 1.17.18 on the same Terminal-Bench 2.1 tasks with GLM-5.2 Max. The task root must match the 89 task ids and canonical task-tree fingerprint of the frozen official revision; a matching Harbor export with one task directory per id is accepted directly. Before model sampling, Harbor's Oracle inspects tasks in the frozen seeded order under the same verifier policy and selects the first 30 that pass. The immutable qualification evidence and selected task ids are bound into the run manifest. Maka keeps active and stale tool-result pruning enabled while semantic compact is explicitly disabled in both the manifest and runtime environment. diff --git a/terminal-bench-smoke/opencode_title_harbor_agent.py b/packages/headless/harbor/opencode_title_harbor_agent.py similarity index 100% rename from terminal-bench-smoke/opencode_title_harbor_agent.py rename to packages/headless/harbor/opencode_title_harbor_agent.py diff --git a/packages/headless/harbor/run-terminal-bench-smoke.mjs b/packages/headless/harbor/run-terminal-bench-smoke.mjs new file mode 100644 index 0000000000..4435c99e48 --- /dev/null +++ b/packages/headless/harbor/run-terminal-bench-smoke.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node + +/** + * Run a structured Terminal-Bench sample job through the local Harbor smoke + * harness. Replaces the retired terminal-bench-smoke/run-terminal-bench-sample.sh + * and run-terminal-bench-sample-heavy.sh shell scripts with a single pure-Node + * entrypoint. Maka profiles drive the authoritative maka_agent:MakaAgent adapter + * in task-run host-bridge mode; heavy-task and autonomous experiments run through + * `--profile maka-heavy` and `--profile maka-heavy-prune`. + */ + +import { spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { delimiter, dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildSmokeJobConfig, resolveSmokeRunTargets } from '#harbor-smoke-config'; + +const HARBOR_DIR = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HARBOR_DIR, '..', '..', '..'); +const MANIFEST_PATH = join(HARBOR_DIR, 'terminal-bench-smoke-profiles.json'); + +const USAGE = `Run a structured Terminal-Bench sample job through the local Harbor smoke harness. + +Usage: + node packages/headless/harbor/run-terminal-bench-smoke.mjs [options] + +Profiles: + maka-basic Maka task-run host bridge, non-autonomous, DeepSeek V4 Pro (default) + maka-heavy Maka task-run heavy-task bridge for trace/evidence experiments + maka-heavy-prune Maka heavy-task bridge with autonomous prior-attempt runtime replay + and stale tool-result archive pruning enabled + maka-prune-default Post-#621 default prune pipeline with continuation (stale A/B B arm) + maka-stale-off maka-prune-default with stale prune explicitly off (stale A/B A arm) + maka-retrieval-on maka-prune-default plus eager archive retrieval (retrieval A/B B arm) + opencode OpenCode Harbor wrapper + oracle Harbor oracle agent for cheap wrapper/dataset smoke tests + +Options: + --profile NAME Run profile (default: maka-basic) + --compare Run comparison profiles sequentially (default: maka-basic,opencode) + --compare-profiles LIST Comma-separated profiles for --compare + --task PATTERN Harbor task pattern (default: *sqlite-with-gcov) + --n-tasks N Pick N tasks instead of using --task + --job-name NAME Harbor job name (default: generated with timestamp) + --model MODEL Override model. For Maka this sets MAKA_MODEL; for OpenCode it sets model_name. + --steps N Override MAKA_MAX_STEPS for Maka profiles + --agent-timeout-sec N Override MAKA_HARBOR_AGENT_TIMEOUT_SEC for Maka profiles + --dataset NAME Override dataset name (default: terminal-bench-sample) + --dataset-version VERSION Override dataset version (default: 2.0) + --dry-run Generate and print config path/command without running Harbor + -h, --help Show this help + +Environment: + HARBOR_BIN Harbor executable (default: harbor on PATH) + +Examples: + node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile oracle --n-tasks 1 + node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-basic --task '*sqlite-with-gcov' + node packages/headless/harbor/run-terminal-bench-smoke.mjs --compare --task '*sqlite-with-gcov' + node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-heavy --compare-profiles maka-heavy,opencode --compare +`; + +function parseArgs(argv) { + const opts = { + profile: 'maka-basic', + compare: false, + compareProfiles: 'maka-basic,opencode', + taskPattern: undefined, + nTasks: undefined, + jobName: undefined, + model: undefined, + maxSteps: undefined, + agentTimeoutSec: undefined, + datasetName: undefined, + datasetVersion: undefined, + dryRun: false, + help: false, + }; + const takeValue = (i, flag) => { + const value = argv[i + 1]; + if (value === undefined) { + throw new Error(`missing value for ${flag}`); + } + return value; + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + switch (arg) { + case '--profile': opts.profile = takeValue(i, arg); i++; break; + case '--compare': opts.compare = true; break; + case '--compare-profiles': opts.compare = true; opts.compareProfiles = takeValue(i, arg); i++; break; + case '--task': opts.taskPattern = takeValue(i, arg); i++; break; + case '--n-tasks': opts.nTasks = Number(takeValue(i, arg)); i++; break; + case '--job-name': opts.jobName = takeValue(i, arg); i++; break; + case '--model': opts.model = takeValue(i, arg); i++; break; + case '--steps': opts.maxSteps = takeValue(i, arg); i++; break; + case '--agent-timeout-sec': opts.agentTimeoutSec = takeValue(i, arg); i++; break; + case '--dataset': opts.datasetName = takeValue(i, arg); i++; break; + case '--dataset-version': opts.datasetVersion = takeValue(i, arg); i++; break; + case '--dry-run': opts.dryRun = true; break; + case '-h': + case '--help': opts.help = true; break; + default: + throw new Error(`unknown option: ${arg}`); + } + } + return opts; +} + +function overridesFor(opts) { + return { + ...(opts.taskPattern !== undefined ? { taskPattern: opts.taskPattern } : {}), + ...(opts.nTasks !== undefined ? { nTasks: opts.nTasks } : {}), + ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.maxSteps !== undefined ? { maxSteps: opts.maxSteps } : {}), + ...(opts.agentTimeoutSec !== undefined ? { agentTimeoutSec: opts.agentTimeoutSec } : {}), + ...(opts.datasetName !== undefined ? { datasetName: opts.datasetName } : {}), + ...(opts.datasetVersion !== undefined ? { datasetVersion: opts.datasetVersion } : {}), + }; +} + +function main() { + let opts; + try { + opts = parseArgs(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`${error.message}\n\n${USAGE}`); + process.exit(2); + } + if (opts.help) { + process.stdout.write(USAGE); + return; + } + if (opts.nTasks !== undefined && (!Number.isInteger(opts.nTasks) || opts.nTasks <= 0)) { + process.stderr.write(`--n-tasks must be a positive integer\n`); + process.exit(2); + } + + const manifest = JSON.parse(readFileSync(MANIFEST_PATH, 'utf8')); + const generatedConfigDir = resolve( + REPO_ROOT, + manifest.defaults?.generatedConfigDir ?? 'packages/headless/harbor/smoke-generated-configs', + ); + mkdirSync(generatedConfigDir, { recursive: true }); + + const harborBin = process.env.HARBOR_BIN || 'harbor'; + const pythonPath = [HARBOR_DIR, process.env.PYTHONPATH].filter(Boolean).join(delimiter); + + const targets = resolveSmokeRunTargets({ + compare: opts.compare, + compareProfiles: opts.compareProfiles, + profile: opts.profile, + jobName: opts.jobName, + }); + + for (const target of targets) { + const { jobName, config } = buildSmokeJobConfig({ + manifest, + profileName: target.profileName, + overrides: { + ...overridesFor(opts), + ...(target.jobName ? { jobName: target.jobName } : {}), + }, + }); + const configPath = join(generatedConfigDir, `${jobName}.json`); + writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8'); + + process.stdout.write(`Generated Harbor config: ${configPath}\n`); + process.stdout.write(`Profile: ${target.profileName}\n`); + process.stdout.write(`Run command:\n`); + process.stdout.write(` PYTHONPATH=${HARBOR_DIR} ${harborBin} run --config ${configPath} --yes\n`); + + if (opts.dryRun) continue; + + const result = spawnSync(harborBin, ['run', '--config', configPath, '--yes'], { + cwd: REPO_ROOT, + stdio: 'inherit', + env: { ...process.env, PYTHONPATH: pythonPath }, + }); + if (result.error) { + process.stderr.write(`failed to launch harbor: ${result.error.message}\n`); + process.exit(1); + } + if (result.status !== 0) { + process.exit(result.status ?? 1); + } + } +} + +main(); diff --git a/packages/headless/harbor/terminal-bench-smoke-profiles.json b/packages/headless/harbor/terminal-bench-smoke-profiles.json new file mode 100644 index 0000000000..1db42fe937 --- /dev/null +++ b/packages/headless/harbor/terminal-bench-smoke-profiles.json @@ -0,0 +1,155 @@ +{ + "schemaVersion": 1, + "description": "Structured Terminal-Bench sample run profiles for the local Harbor smoke harness. Maka profiles drive the authoritative maka_agent:MakaAgent adapter in task-run host-bridge mode (MAKA_HARBOR_MODE=task-run).", + "defaults": { + "jobsDir": "packages/headless/harbor/smoke-jobs", + "generatedConfigDir": "packages/headless/harbor/smoke-generated-configs", + "dataset": { + "name": "terminal-bench-sample", + "version": "2.0" + }, + "taskPattern": "*sqlite-with-gcov", + "nAttempts": 1, + "nConcurrentTrials": 1, + "timeoutMultiplier": 1.0, + "retryMaxRetries": 0, + "modelExtraInstructionPaths": [] + }, + "profiles": { + "maka-basic": { + "description": "Maka task-run host bridge against terminal-bench-sample, matching the successful sqlite-with-gcov DeepSeek V4 Pro sample run shape.", + "agentTimeoutMultiplier": 4.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "0", + "MAKA_HARBOR_AUTONOMOUS": "0", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "80", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" + } + } + }, + "maka-heavy": { + "description": "Maka task-run heavy-task bridge for public sample trace/evidence experiments.", + "agentTimeoutMultiplier": 8.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "1", + "MAKA_HARBOR_AUTONOMOUS": "0", + "MAKA_HEAVY_TASK_MODE": "1", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "100", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "7200" + } + } + }, + "maka-heavy-prune": { + "description": "Maka heavy-task bridge with autonomous prior-attempt runtime replay and stale tool-result archive pruning enabled.", + "agentTimeoutMultiplier": 8.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "1", + "MAKA_HARBOR_AUTONOMOUS": "1", + "MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT": "1", + "MAKA_HEAVY_TASK_MODE": "1", + "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE": "on", + "MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_ESTIMATED_TOKENS": "2048", + "MAKA_CONTEXT_STALE_TOOL_RESULT_MIN_RECENT_TURNS_FULL": "0", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_PRUNE": "on", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MAX_ESTIMATED_TOKENS": "2048", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MIN_STEP_NUMBER": "1", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT": "on", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_STEP_NUMBER": "2", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_RATIO": "0.5", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_ACTIVE_ESTIMATED_TOKENS": "16384", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_MESSAGES": "4", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_SUMMARY_ESTIMATED_TOKENS": "1024", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "100", + "MAKA_AUTONOMOUS_MAX_ATTEMPTS": "3", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "7200" + } + } + }, + "maka-prune-default": { + "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B.", + "agentTimeoutMultiplier": 4.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "0", + "MAKA_HARBOR_AUTONOMOUS": "0", + "MAKA_HARBOR_CONTINUATION": "on", + "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", + "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "50", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" + } + } + }, + "maka-stale-off": { + "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B.", + "agentTimeoutMultiplier": 4.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "0", + "MAKA_HARBOR_AUTONOMOUS": "0", + "MAKA_HARBOR_CONTINUATION": "on", + "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", + "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", + "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE": "off", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "50", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" + } + } + }, + "maka-retrieval-on": { + "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B.", + "agentTimeoutMultiplier": 4.0, + "agent": { + "importPath": "maka_agent:MakaAgent", + "env": { + "MAKA_HARBOR_MODE": "task-run", + "MAKA_HARBOR_USE_TASK_RUN": "0", + "MAKA_HARBOR_AUTONOMOUS": "0", + "MAKA_HARBOR_CONTINUATION": "on", + "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", + "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", + "MAKA_CONTEXT_ARCHIVE_RETRIEVAL": "on", + "MAKA_MODEL": "deepseek-v4-pro", + "MAKA_MAX_STEPS": "50", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" + } + } + }, + "opencode": { + "description": "OpenCode Harbor wrapper against terminal-bench-sample for Maka/OpenCode comparison.", + "agentTimeoutMultiplier": 4.0, + "agent": { + "importPath": "opencode_title_harbor_agent:OpenCodeTitleAgent", + "modelName": "deepseek/deepseek-v4-pro", + "env": {} + } + }, + "oracle": { + "description": "Harbor built-in oracle agent for cheap wrapper/dataset smoke tests.", + "agentTimeoutMultiplier": null, + "extraInstructionPaths": [], + "agent": { + "name": "oracle", + "env": {} + } + } + } +} diff --git a/packages/headless/package.json b/packages/headless/package.json index 9e92caf701..bbf16089af 100644 --- a/packages/headless/package.json +++ b/packages/headless/package.json @@ -35,6 +35,7 @@ "#ab-manifest": "./dist/ab-manifest.js", "#prompt-ab-run": "./dist/prompt-ab-run.js", "#harbor-task-runner": "./dist/harbor-task-runner.js", + "#harbor-smoke-config": "./dist/harbor-smoke-config.js", "#provider-env": "./dist/provider-env.js", "#harbor-cell": "./dist/harbor-cell.js", "#opencode-toolchain": "./dist/opencode-toolchain.js", diff --git a/packages/headless/src/__tests__/harbor-smoke-config.test.ts b/packages/headless/src/__tests__/harbor-smoke-config.test.ts new file mode 100644 index 0000000000..c8348492c2 --- /dev/null +++ b/packages/headless/src/__tests__/harbor-smoke-config.test.ts @@ -0,0 +1,162 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { describe, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + buildSmokeJobConfig, + resolveSmokeRunTargets, + type SmokeManifest, +} from '../harbor-smoke-config.js'; + +const repoRoot = resolve(fileURLToPath(new URL('../../../..', import.meta.url))); + +async function loadManifest(): Promise { + const path = resolve(repoRoot, 'packages/headless/harbor/terminal-bench-smoke-profiles.json'); + return JSON.parse(await readFile(path, 'utf8')) as SmokeManifest; +} + +const fixedNow = () => new Date('2026-07-16T12:34:56.000Z'); + +describe('harbor smoke config generation', () => { + test('unknown profile throws with available names', async () => { + const manifest = await loadManifest(); + assert.throws( + () => buildSmokeJobConfig({ manifest, profileName: 'does-not-exist' }), + /unknown profile "does-not-exist"\. Available profiles: .*maka-basic/, + ); + }); + + test('maka profiles drive maka_agent:MakaAgent in task-run mode and tag the dataset', async () => { + const manifest = await loadManifest(); + for (const profileName of [ + 'maka-basic', + 'maka-heavy', + 'maka-heavy-prune', + 'maka-prune-default', + 'maka-stale-off', + 'maka-retrieval-on', + ]) { + const { config } = buildSmokeJobConfig({ manifest, profileName, overrides: { jobName: `job-${profileName}` } }); + const agent = (config.agents as Array>)[0]!; + const env = agent.env as Record; + assert.equal(agent.import_path, 'maka_agent:MakaAgent', profileName); + assert.equal(env.MAKA_HARBOR_MODE, 'task-run', profileName); + assert.equal(env.MAKA_BENCHMARK_DATASET, 'terminal-bench-sample', profileName); + const datasets = config.datasets as Array>; + assert.equal(datasets[0]!.name, 'terminal-bench-sample', profileName); + } + }); + + test('heavy profile preserves heavy-task env verbatim', async () => { + const manifest = await loadManifest(); + const { config } = buildSmokeJobConfig({ manifest, profileName: 'maka-heavy', overrides: { jobName: 'job' } }); + const env = (config.agents as Array>)[0]!.env as Record; + assert.equal(env.MAKA_HEAVY_TASK_MODE, '1'); + assert.equal(env.MAKA_HARBOR_USE_TASK_RUN, '1'); + assert.equal(env.MAKA_MAX_STEPS, '100'); + assert.equal(env.MAKA_HARBOR_AGENT_TIMEOUT_SEC, '7200'); + assert.equal(config.agent_timeout_multiplier, 8); + }); + + test('--model override targets MAKA_MODEL for maka and model_name for non-maka', async () => { + const manifest = await loadManifest(); + const maka = buildSmokeJobConfig({ + manifest, + profileName: 'maka-basic', + overrides: { jobName: 'j', model: 'deepseek/deepseek-vX' }, + }); + const makaAgent = (maka.config.agents as Array>)[0]!; + assert.equal((makaAgent.env as Record).MAKA_MODEL, 'deepseek/deepseek-vX'); + assert.equal(makaAgent.model_name, null); + + const opencode = buildSmokeJobConfig({ + manifest, + profileName: 'opencode', + overrides: { jobName: 'j', model: 'deepseek/other' }, + }); + const ocAgent = (opencode.config.agents as Array>)[0]!; + assert.equal(ocAgent.model_name, 'deepseek/other'); + assert.equal(ocAgent.import_path, 'opencode_title_harbor_agent:OpenCodeTitleAgent'); + assert.deepEqual(ocAgent.env, {}); + }); + + test('n-tasks replaces task_names with a task count', async () => { + const manifest = await loadManifest(); + const withPattern = buildSmokeJobConfig({ manifest, profileName: 'oracle', overrides: { jobName: 'j', taskPattern: '*foo' } }); + const withCount = buildSmokeJobConfig({ manifest, profileName: 'oracle', overrides: { jobName: 'j', nTasks: 3 } }); + const dsPattern = (withPattern.config.datasets as Array>)[0]!; + const dsCount = (withCount.config.datasets as Array>)[0]!; + assert.deepEqual(dsPattern.task_names, ['*foo']); + assert.equal(dsPattern.n_tasks, null); + assert.equal(dsCount.task_names, null); + assert.equal(dsCount.n_tasks, 3); + }); + + test('rejects non-positive n-tasks', async () => { + const manifest = await loadManifest(); + assert.throws( + () => buildSmokeJobConfig({ manifest, profileName: 'oracle', overrides: { jobName: 'j', nTasks: 0 } }), + /--n-tasks must be a positive integer/, + ); + }); + + test('dataset name/version overrides flow into the dataset and MAKA_BENCHMARK_DATASET', async () => { + const manifest = await loadManifest(); + const { config } = buildSmokeJobConfig({ + manifest, + profileName: 'maka-basic', + overrides: { jobName: 'j', datasetName: 'terminal-bench', datasetVersion: '3.1' }, + }); + const ds = (config.datasets as Array>)[0]!; + assert.equal(ds.name, 'terminal-bench'); + assert.equal(ds.version, '3.1'); + const env = (config.agents as Array>)[0]!.env as Record; + assert.equal(env.MAKA_BENCHMARK_DATASET, 'terminal-bench'); + }); + + test('oracle profile keeps the built-in agent and null import path', async () => { + const manifest = await loadManifest(); + const { config } = buildSmokeJobConfig({ manifest, profileName: 'oracle', overrides: { jobName: 'j' } }); + const agent = (config.agents as Array>)[0]!; + assert.equal(agent.name, 'oracle'); + assert.equal(agent.import_path, null); + assert.equal(config.agent_timeout_multiplier, null); + }); + + test('generated job name uses the injected clock when no explicit name is given', () => { + const manifest: SmokeManifest = { + defaults: { taskPattern: '*sqlite-with-gcov' }, + profiles: { 'maka-basic': { agent: { importPath: 'maka_agent:MakaAgent', env: {} } } }, + }; + const { jobName } = buildSmokeJobConfig({ manifest, profileName: 'maka-basic', overrides: { now: fixedNow } }); + assert.equal(jobName, 'maka-basic-terminal-bench-sample-sqlite-with-gcov-20260716T123456Z'); + }); + + test('resolveSmokeRunTargets returns a single target without compare', () => { + assert.deepEqual( + resolveSmokeRunTargets({ compare: false, profile: 'maka-heavy', jobName: 'run1' }), + [{ profileName: 'maka-heavy', jobName: 'run1' }], + ); + }); + + test('resolveSmokeRunTargets splits compare profiles and suffixes job names', () => { + assert.deepEqual( + resolveSmokeRunTargets({ compare: true, compareProfiles: 'maka-heavy, opencode', profile: 'x', jobName: 'run1' }), + [ + { profileName: 'maka-heavy', jobName: 'run1-maka-heavy' }, + { profileName: 'opencode', jobName: 'run1-opencode' }, + ], + ); + }); + + test('resolveSmokeRunTargets leaves job names blank when none is supplied', () => { + assert.deepEqual( + resolveSmokeRunTargets({ compare: true, compareProfiles: 'maka-basic,opencode', profile: 'x' }), + [ + { profileName: 'maka-basic', jobName: '' }, + { profileName: 'opencode', jobName: '' }, + ], + ); + }); +}); diff --git a/packages/headless/src/harbor-smoke-config.ts b/packages/headless/src/harbor-smoke-config.ts new file mode 100644 index 0000000000..4499af8f4d --- /dev/null +++ b/packages/headless/src/harbor-smoke-config.ts @@ -0,0 +1,263 @@ +/** + * Terminal-Bench smoke job-config generation. + * + * This is the registry-dataset / external-verifier / host-bridge Harbor config + * shape used by the local smoke harness (run-terminal-bench-smoke.mjs). It is + * deliberately distinct from buildHarborJobConfig in harbor-task-runner.ts, + * which builds the fixed-prompt controller's local-task + custom-verifier + + * in-container-cell config. The two config schemas do not overlap, so they stay + * separate rather than being forced through one parameterized builder. + * + * Ported verbatim (behaviour-for-behaviour) from the embedded Node generator in + * the retired terminal-bench-smoke/run-terminal-bench-sample.sh so existing + * profiles keep producing byte-equivalent configs. + */ + +export interface SmokeManifestDataset { + name?: string; + version?: string; +} + +export interface SmokeManifestDefaults { + jobsDir?: string; + generatedConfigDir?: string; + dataset?: SmokeManifestDataset; + taskPattern?: string; + nAttempts?: number; + nConcurrentTrials?: number; + timeoutMultiplier?: number; + retryMaxRetries?: number; + modelExtraInstructionPaths?: string[]; +} + +export interface SmokeManifestAgent { + name?: string; + importPath?: string; + modelName?: string; + env?: Record; + kwargs?: Record; +} + +export interface SmokeManifestProfile { + description?: string; + agentTimeoutMultiplier?: number | null; + extraInstructionPaths?: string[]; + agent?: SmokeManifestAgent; +} + +export interface SmokeManifest { + schemaVersion?: number; + description?: string; + defaults?: SmokeManifestDefaults; + profiles?: Record; +} + +export interface SmokeConfigOverrides { + taskPattern?: string; + jobName?: string; + model?: string; + maxSteps?: string; + agentTimeoutSec?: string; + nTasks?: number; + datasetName?: string; + datasetVersion?: string; + benchmarkDataset?: string; + /** Injectable clock for the generated job-name timestamp (defaults to now). */ + now?: () => Date; +} + +export interface SmokeJobConfigResult { + jobName: string; + config: Record; +} + +export interface SmokeRunTarget { + profileName: string; + /** Empty string means "let buildSmokeJobConfig generate a timestamped name". */ + jobName: string; +} + +function slug(value: string): string { + return ( + String(value) + .replace(/^\*/, '') + .replace(/[^A-Za-z0-9_.-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'sample' + ); +} + +function retryConfig(defaults: SmokeManifestDefaults): Record { + return { + max_retries: Number(defaults.retryMaxRetries || 0), + include_exceptions: null, + exclude_exceptions: [ + 'AgentTimeoutError', + 'VerifierOutputParseError', + 'VerifierTimeoutError', + 'RewardFileNotFoundError', + 'RewardFileEmptyError', + ], + wait_multiplier: 1.0, + min_wait_sec: 1.0, + max_wait_sec: 60.0, + }; +} + +/** Split a smoke run into ordered profile/job-name targets, matching the + * retired shell runner's --compare / single-profile semantics. */ +export function resolveSmokeRunTargets(input: { + compare: boolean; + compareProfiles?: string; + profile: string; + jobName?: string; +}): SmokeRunTarget[] { + if (!input.compare) { + return [{ profileName: input.profile, jobName: input.jobName ?? '' }]; + } + const list = (input.compareProfiles ?? 'maka-basic,opencode').split(','); + const targets: SmokeRunTarget[] = []; + for (const raw of list) { + const profileName = raw.trim(); + if (!profileName) continue; + targets.push({ + profileName, + jobName: input.jobName ? `${input.jobName}-${profileName}` : '', + }); + } + return targets; +} + +export function buildSmokeJobConfig(input: { + manifest: SmokeManifest; + profileName: string; + overrides?: SmokeConfigOverrides; +}): SmokeJobConfigResult { + const { manifest, profileName } = input; + const overrides = input.overrides ?? {}; + const defaults = manifest.defaults ?? {}; + const profile = manifest.profiles?.[profileName]; + if (!profile) { + const names = Object.keys(manifest.profiles ?? {}).join(', '); + throw new Error(`unknown profile "${profileName}". Available profiles: ${names}`); + } + + const taskPattern = overrides.taskPattern || defaults.taskPattern || '*sqlite-with-gcov'; + const datasetName = overrides.datasetName || defaults.dataset?.name || 'terminal-bench-sample'; + const datasetVersion = overrides.datasetVersion || defaults.dataset?.version || '2.0'; + const nTasks = overrides.nTasks ?? null; + if (nTasks !== null && (!Number.isInteger(nTasks) || nTasks <= 0)) { + throw new Error(`--n-tasks must be a positive integer, got ${nTasks}`); + } + + const now = overrides.now ?? (() => new Date()); + const jobName = + overrides.jobName || + [ + profileName, + 'terminal-bench-sample', + nTasks ? `n${nTasks}` : slug(taskPattern), + now().toISOString().replace(/[-:]/g, '').replace(/\..+$/, 'Z'), + ].join('-'); + + const agent = profile.agent ?? {}; + const agentEnv: Record = { ...(agent.env ?? {}) }; + let agentModelName = agent.modelName ?? null; + const modelOverride = overrides.model; + if (modelOverride) { + if (profileName.startsWith('maka-')) { + agentEnv.MAKA_MODEL = modelOverride; + } else { + agentModelName = modelOverride; + } + } + + if (overrides.maxSteps) agentEnv.MAKA_MAX_STEPS = overrides.maxSteps; + if (overrides.agentTimeoutSec) agentEnv.MAKA_HARBOR_AGENT_TIMEOUT_SEC = overrides.agentTimeoutSec; + if (profileName.startsWith('maka-') && !agentEnv.MAKA_BENCHMARK_DATASET) { + agentEnv.MAKA_BENCHMARK_DATASET = overrides.benchmarkDataset || datasetName; + } + + const extraInstructionPaths = Object.prototype.hasOwnProperty.call(profile, 'extraInstructionPaths') + ? profile.extraInstructionPaths ?? [] + : defaults.modelExtraInstructionPaths ?? []; + + const config: Record = { + job_name: jobName, + jobs_dir: defaults.jobsDir || 'packages/headless/harbor/smoke-jobs', + n_attempts: Number(defaults.nAttempts || 1), + timeout_multiplier: Number(defaults.timeoutMultiplier || 1.0), + agent_timeout_multiplier: + profile.agentTimeoutMultiplier === undefined || profile.agentTimeoutMultiplier === null + ? null + : profile.agentTimeoutMultiplier, + verifier_timeout_multiplier: null, + agent_setup_timeout_multiplier: null, + environment_build_timeout_multiplier: null, + debug: false, + n_concurrent_trials: Number(defaults.nConcurrentTrials || 1), + quiet: false, + retry: retryConfig(defaults), + environment: { + type: 'docker', + import_path: null, + force_build: false, + delete: true, + cpu_enforcement_policy: 'auto', + memory_enforcement_policy: 'auto', + override_cpus: null, + override_memory_mb: null, + override_storage_mb: null, + override_gpus: null, + override_tpu: null, + mounts: null, + extra_docker_compose: [], + env: {}, + kwargs: {}, + extra_allowed_hosts: [], + }, + verifier: { + override_timeout_sec: null, + max_timeout_sec: null, + env: {}, + disable: false, + }, + metrics: [], + agents: [ + { + name: agent.name ?? null, + import_path: agent.importPath ?? null, + model_name: agentModelName, + skills: [], + override_timeout_sec: null, + override_setup_timeout_sec: null, + max_timeout_sec: null, + extra_allowed_hosts: [], + kwargs: agent.kwargs ?? {}, + env: agentEnv, + mcp_servers: [], + }, + ], + datasets: [ + { + path: null, + name: datasetName, + version: datasetVersion, + ref: null, + registry_url: null, + registry_path: null, + overwrite: false, + download_dir: null, + task_names: nTasks ? null : [taskPattern], + exclude_task_names: null, + n_tasks: nTasks, + }, + ], + tasks: [], + artifacts: [], + extra_instruction_paths: extraInstructionPaths, + plugins: [], + }; + + return { jobName, config }; +} diff --git a/packages/headless/src/index.ts b/packages/headless/src/index.ts index c471a1ab08..9bd099ac3c 100644 --- a/packages/headless/src/index.ts +++ b/packages/headless/src/index.ts @@ -4,6 +4,20 @@ // package-local entrypoints, not the root API. Minimal usage is // `runExperiment(config, task, { storageRoot })`. export { runPromptOptimizationRun } from './prompt-optimization-run.js'; +export { + buildSmokeJobConfig, + resolveSmokeRunTargets, +} from './harbor-smoke-config.js'; +export type { + SmokeManifest, + SmokeManifestAgent, + SmokeManifestDataset, + SmokeManifestDefaults, + SmokeManifestProfile, + SmokeConfigOverrides, + SmokeJobConfigResult, + SmokeRunTarget, +} from './harbor-smoke-config.js'; export type { MakaChangeAuditRecord } from './change-audit.js'; export type { PromptOptimizationRunInput, From 91d11008916b309b00dbe5245ffb41dae19690c6 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 17:49:00 +0800 Subject: [PATCH 2/8] feat(headless): add task-run host-bridge mode to the Maka Harbor adapter MAKA_HARBOR_MODE=task-run makes maka_agent:MakaAgent run the full task-run controller on the host and bridge tool execution into the task container via the shared _ToolExecutorServer, spawning `node dist/cli.js harbor run --mode task-run --isolation harbor-http`. Ports the retired terminal-bench-smoke fork's env normalization, --autonomous/--heavy-task derivation, status/stdout artifacts, last-line JSON result parsing, benchmark-vs-infra exit policy, and the direct-make-mips smoke branch. Default mode stays cell (unchanged). --- packages/headless/harbor/maka_agent.py | 730 +++++++++++++++++++++++++ 1 file changed, 730 insertions(+) diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index f785e52045..4b41cf73d9 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -4,11 +4,13 @@ import asyncio import concurrent.futures +import contextlib import json import os import secrets import shlex import threading +from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any @@ -26,6 +28,20 @@ _COMMAND_SCOPE_ENV = "MAKA_HARBOR_COMMAND_SCOPE" _COMMAND_SCOPE_ROOT = "/tmp/maka-harbor-command-scopes" +# Location of this adapter and the repo it ships in. The task-run host mode +# spawns `node /packages/headless/dist/cli.js` on the host, so it needs to +# find the built headless CLI relative to this file when no explicit repo root +# env is set. +_HARBOR_DIR = Path(__file__).resolve().parent +_HEADLESS_DIR = _HARBOR_DIR.parent +_REPO_ROOT_DEFAULT = _HEADLESS_DIR.parent.parent +_DEFAULT_RUNNER_ENV = Path( + os.environ.get( + "MAKA_HARBOR_RUNNER_ENV_FILE", + str(Path.home() / ".config" / "maka" / "harbor-runner.env"), + ) +) + _HOST_NODE_ENV_ALLOWLIST = { "PATH", @@ -103,10 +119,35 @@ class MakaAgent(BaseInstalledAgent): def name() -> str: return "maka" + def _harbor_mode(self) -> str: + """cell (default) runs a RuntimeRunner cell; task-run runs the full + task-run controller on the host and bridges tool execution into the + container via the shared _ToolExecutorServer. The mode switch keeps the + heavy-task / autonomous experiment path that used to live in the + terminal-bench-smoke fork on the single authoritative adapter.""" + mode = (self._get_env("MAKA_HARBOR_MODE") or "cell").strip() + if mode not in ("cell", "task-run"): + raise RuntimeError(f"MAKA_HARBOR_MODE must be cell or task-run, got {mode!r}") + return mode + def get_version_command(self) -> str | None: + # task-run runs Maka on the host, not in the container, so there is no + # in-container binary to version-check. + if self._harbor_mode() == "task-run": + return None return "node --version" async def install(self, environment: BaseEnvironment) -> None: + if self._harbor_mode() == "task-run": + # Host-bridge task-run: node and the headless CLI run on the host and + # bridge tool execution into the task container, so nothing installs + # inside the task. Fail fast if the built CLI is missing. + cli_path = self._headless_cli_path() + if not cli_path.is_file(): + raise RuntimeError( + f"headless CLI not built at {cli_path}; run `npm run build` in packages/headless" + ) + return maka_repo = self._resolved_flags.get("maka_repo", "/opt/maka-agent") self._harbor_backend() run_cell = (Path(maka_repo) / "packages" / "headless" / "harbor" / "run-cell.mjs").as_posix() @@ -165,6 +206,9 @@ async def run( environment: BaseEnvironment, context: AgentContext, ) -> None: + if self._harbor_mode() == "task-run": + await self._run_task_run_host(instruction, environment, context) + return agent_dir = EnvironmentPaths.agent_dir await self.exec_as_agent(environment, command=f"mkdir -p {agent_dir.as_posix()}") @@ -201,6 +245,13 @@ def _run_host_cell_path(self) -> str: maka_repo = self._get_env("MAKA_HOST_REPO_ROOT") or os.getcwd() return (Path(maka_repo) / "packages" / "headless" / "harbor" / "run-host-cell.mjs").as_posix() + def _host_repo_root(self) -> Path: + override = self._get_env("MAKA_HOST_REPO_ROOT") or self._get_env("MAKA_REPO_DIR") + return Path(override) if override else _REPO_ROOT_DEFAULT + + def _headless_cli_path(self) -> Path: + return self._host_repo_root() / "packages" / "headless" / "dist" / "cli.js" + _DEFAULT_CELL_TIMEOUT_SEC = 900 _DEFAULT_CELL_SETTLEMENT_GRACE_SEC = 30 @@ -458,6 +509,427 @@ def _write_trajectory(self, output: dict[str, Any]) -> None: self.logger.debug("Could not write Maka trajectory %s: %s", trajectory_path, exc) + async def _run_task_run_host( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + """Run the full task-run controller on the host, bridging tool execution + into the task container. Ported from the terminal-bench-smoke fork so the + heavy-task / autonomous experiment path is preserved end to end.""" + self.logs_dir.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env.update(_load_env_file(_DEFAULT_RUNNER_ENV)) + env.update(getattr(self, "_extra_env", {}) or {}) + _normalize_cli_env(env) + env.setdefault("MAKA_REPO_DIR", str(self._host_repo_root())) + env.setdefault("MAKA_MODEL", "deepseek-chat") + env.setdefault("MAKA_MAX_STEPS", "35") + env.setdefault("MAKA_TASK_RUN_OUT_DIR", str(self.logs_dir / "maka-task-run")) + task_run_out_dir = Path(env["MAKA_TASK_RUN_OUT_DIR"]) + if not task_run_out_dir.is_absolute(): + task_run_out_dir = task_run_out_dir.resolve() + env["MAKA_TASK_RUN_OUT_DIR"] = str(task_run_out_dir) + # _normalize_cli_env derives MAKA_OUTPUT_DIR/MAKA_STORAGE_ROOT from + # MAKA_TASK_RUN_OUT_DIR; re-apply now that the out dir is finalized. + env.setdefault("MAKA_OUTPUT_DIR", str(task_run_out_dir)) + env.setdefault("MAKA_STORAGE_ROOT", str(task_run_out_dir / "runs")) + + task_workdir, workdir_probe = await self._resolve_task_workdir(environment) + + stdout_path = self.logs_dir / "maka-harbor.stdout.json" + stderr_path = self.logs_dir / "maka-harbor.stderr.log" + status_path = self.logs_dir / "maka-harbor.status.json" + instruction_path = self.logs_dir / "instruction.txt" + started_at = _utc_now() + task_run_out_dir.mkdir(parents=True, exist_ok=True) + instruction_path.write_text(instruction, encoding="utf-8") + stdout_path.write_bytes(b"") + stderr_path.write_bytes(b"") + self._write_status( + status_path, + { + "status": "starting", + "startedAt": started_at, + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + }, + ) + + if env.get("MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE") == "1": + await self._run_direct_make_mips_smoke( + environment, + context=context, + env=env, + task_workdir=task_workdir, + workdir_probe=workdir_probe, + stdout_path=stdout_path, + stderr_path=stderr_path, + status_path=status_path, + task_run_out_dir=task_run_out_dir, + started_at=started_at, + ) + return + + timeout_sec = int(env.get("MAKA_HARBOR_AGENT_TIMEOUT_SEC", "1800")) + proc: asyncio.subprocess.Process | None = None + async with _ToolExecutorServer(self, environment) as executor: + env["MAKA_HARBOR_TOOL_EXECUTOR_URL"] = executor.url + env["MAKA_HARBOR_TOOL_EXECUTOR_TOKEN"] = executor.token + command = _headless_harbor_command( + cli_path=self._headless_cli_path(), + instruction_path=instruction_path, + task_workdir=task_workdir, + task_id=str(getattr(environment, "session_id", None) or env.get("MAKA_TASK_ID") or "terminal-bench-task"), + out_dir=task_run_out_dir, + env=env, + ) + try: + proc = await asyncio.create_subprocess_exec( + *command, + cwd=str(self._host_repo_root()), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + self._write_status( + status_path, + { + "status": "running", + "startedAt": started_at, + "updatedAt": _utc_now(), + "runnerPid": proc.pid, + "timeoutSec": timeout_sec, + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + "command": _redacted_command(command), + }, + ) + stdout, stderr = await self._communicate_streaming( + proc=proc, + stdin_payload=b"", + stdout_path=stdout_path, + stderr_path=stderr_path, + timeout_sec=timeout_sec, + ) + except asyncio.TimeoutError: + self._write_status( + status_path, + { + "status": "timeout", + "startedAt": started_at, + "finishedAt": _utc_now(), + "runnerPid": proc.pid if proc else None, + "returnCode": proc.returncode if proc else None, + "timeoutSec": timeout_sec, + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + "command": _redacted_command(command), + }, + ) + raise + except Exception as exc: + self._write_status( + status_path, + { + "status": "failed", + "startedAt": started_at, + "finishedAt": _utc_now(), + "runnerPid": proc.pid if proc else None, + "returnCode": proc.returncode if proc else None, + "error": str(exc), + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + "command": _redacted_command(command), + }, + ) + raise + + parsed = self._parse_node_result(stdout) + assert proc is not None + self._write_status( + status_path, + { + "status": "completed" if proc.returncode == 0 else "failed", + "startedAt": started_at, + "finishedAt": _utc_now(), + "runnerPid": proc.pid, + "returnCode": proc.returncode, + "parsedStatus": parsed.get("status"), + "benchmarkFailureKind": parsed.get("benchmarkFailureKind"), + "benchmarkFailureShouldThrow": parsed.get("benchmarkFailureShouldThrow"), + "stdoutBytes": len(stdout), + "stderrBytes": len(stderr), + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + "command": _redacted_command(command), + }, + ) + context.metadata = { + **(context.metadata or {}), + "maka_harbor": { + "return_code": proc.returncode, + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + "status": parsed.get("status"), + "model": parsed.get("model"), + "max_steps": parsed.get("maxSteps"), + "autonomous": parsed.get("autonomous"), + "autonomous_max_attempts": parsed.get("autonomousMaxAttempts"), + "autonomous_max_runtime_steps": parsed.get("autonomousMaxRuntimeSteps"), + "autonomous_max_wall_time_ms": parsed.get("autonomousMaxWallTimeMs"), + "event_count": parsed.get("eventCount"), + "message_count": parsed.get("messageCount"), + "llm_call_count": parsed.get("llmCallCount"), + "tool_call_count": parsed.get("toolCallCount"), + "error": parsed.get("error"), + "benchmark_failure_kind": parsed.get("benchmarkFailureKind"), + "benchmark_failure_should_throw": parsed.get("benchmarkFailureShouldThrow"), + "task_run": parsed.get("taskRun") or _task_run_summary(parsed), + "resolved_cwd": task_workdir, + "workdir_probe": workdir_probe, + }, + } + usage = parsed.get("tokenUsage") if isinstance(parsed, dict) else None + if isinstance(usage, dict): + context.n_input_tokens = _int_or_none(usage.get("input")) + context.n_cache_tokens = _int_or_none(usage.get("cacheHitInput")) + context.n_output_tokens = _int_or_none(usage.get("output")) + + if proc.returncode != 0: + raise RuntimeError(f"Maka Harbor task-run failed; see {stderr_path}") + + async def _run_direct_make_mips_smoke( + self, + environment: BaseEnvironment, + *, + context: AgentContext, + env: dict[str, str], + task_workdir: str, + workdir_probe: list[dict[str, Any]], + stdout_path: Path, + stderr_path: Path, + status_path: Path, + task_run_out_dir: Path, + started_at: str, + ) -> None: + result = await self.exec_as_agent( + environment, + command=_direct_make_mips_smoke_command(), + cwd=task_workdir, + timeout_sec=30, + ) + return_code = _exec_exit_code(result) + direct_payload = { + "ok": return_code == 0, + "status": "completed" if return_code == 0 else "failed", + "mode": "direct-make-mips-smoke", + "cwd": task_workdir, + "returnCode": return_code, + "stdout": _exec_stdout(result), + "stderr": _exec_stderr(result), + } + stdout_path.write_text(json.dumps(direct_payload) + "\n", encoding="utf-8") + stderr_path.write_text("", encoding="utf-8") + self._write_status( + status_path, + { + "status": direct_payload["status"], + "startedAt": started_at, + "finishedAt": _utc_now(), + "mode": direct_payload["mode"], + "returnCode": return_code, + "stdoutLog": str(stdout_path), + "stderrLog": str(stderr_path), + "taskRunOutDir": str(task_run_out_dir), + "resolvedCwd": task_workdir, + "workdirProbe": workdir_probe, + "runnerEnv": _runner_env_summary(env), + }, + ) + context.metadata = { + **(context.metadata or {}), + "maka_harbor": { + "return_code": return_code, + "stdout_log": str(stdout_path), + "stderr_log": str(stderr_path), + "status": direct_payload["status"], + "model": "direct-make-mips-smoke", + "max_steps": 0, + "event_count": 0, + "message_count": 0, + "llm_call_count": 0, + "tool_call_count": 1, + "error": None if return_code == 0 else _exec_stderr(result), + "resolved_cwd": task_workdir, + "workdir_probe": workdir_probe, + }, + } + if return_code != 0: + raise RuntimeError(f"direct make-mips smoke setup failed; see {stdout_path}") + + async def _resolve_task_workdir( + self, + environment: BaseEnvironment, + ) -> tuple[str, list[dict[str, Any]]]: + configured = getattr(getattr(environment, "task_env_config", None), "workdir", None) + candidates: list[str | None] = [] + if configured: + candidates.append(str(configured)) + candidates.extend([None, "/app", "/workspace", "/"]) + + seen: set[str] = set() + probes: list[dict[str, Any]] = [] + for candidate in candidates: + marker = "" if candidate is None else candidate + if marker in seen: + continue + seen.add(marker) + result = await self.exec_as_agent( + environment, + command="pwd", + cwd=candidate, + timeout_sec=10, + ) + stdout = _exec_stdout(result).strip() + stderr = _exec_stderr(result).strip() + return_code = _exec_exit_code(result) + probes.append( + { + "candidate": marker, + "return_code": return_code, + "stdout": stdout, + "stderr": stderr, + } + ) + if return_code == 0: + resolved = _last_absolute_path(stdout) + if resolved: + return resolved, probes + + fallback = str(configured or "/") + probes.append({"fallback": fallback}) + return fallback, probes + + async def _communicate_streaming( + self, + *, + proc: asyncio.subprocess.Process, + stdin_payload: bytes, + stdout_path: Path, + stderr_path: Path, + timeout_sec: int, + ) -> tuple[bytes, bytes]: + stdout_chunks: list[bytes] = [] + stderr_chunks: list[bytes] = [] + stdin_task = asyncio.create_task(self._write_process_stdin(proc, stdin_payload)) + stdout_task = asyncio.create_task(self._tee_stream(proc.stdout, stdout_path, stdout_chunks)) + stderr_task = asyncio.create_task(self._tee_stream(proc.stderr, stderr_path, stderr_chunks)) + + try: + await asyncio.wait_for(proc.wait(), timeout=timeout_sec) + await asyncio.wait_for( + asyncio.gather(stdin_task, stdout_task, stderr_task), + timeout=30, + ) + except asyncio.TimeoutError: + with stderr_path.open("ab") as handle: + marker = { + "event": "maka_harbor_timeout", + "timeoutSec": timeout_sec, + "at": _utc_now(), + } + handle.write(("\n" + json.dumps(marker) + "\n").encode("utf-8")) + handle.flush() + if proc.returncode is None: + proc.kill() + with contextlib.suppress(Exception): + await asyncio.wait_for(proc.wait(), timeout=10) + raise + finally: + for task in (stdin_task, stdout_task, stderr_task): + if not task.done(): + task.cancel() + await asyncio.gather(stdin_task, stdout_task, stderr_task, return_exceptions=True) + + return b"".join(stdout_chunks), b"".join(stderr_chunks) + + @staticmethod + async def _write_process_stdin( + proc: asyncio.subprocess.Process, + payload: bytes, + ) -> None: + if proc.stdin is None: + return + try: + proc.stdin.write(payload) + await proc.stdin.drain() + except (BrokenPipeError, ConnectionResetError): + pass + finally: + with contextlib.suppress(Exception): + proc.stdin.close() + with contextlib.suppress(Exception): + await proc.stdin.wait_closed() + + @staticmethod + async def _tee_stream( + reader: asyncio.StreamReader | None, + path: Path, + chunks: list[bytes], + ) -> None: + if reader is None: + return + with path.open("ab") as handle: + while True: + chunk = await reader.read(65536) + if not chunk: + break + chunks.append(chunk) + handle.write(chunk) + handle.flush() + + @staticmethod + def _write_status(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + @staticmethod + def _parse_node_result(stdout: bytes) -> dict[str, Any]: + text = stdout.decode("utf-8", errors="replace").strip() + if not text: + return {} + # The runner writes one JSON object to stdout. If a dependency writes + # noise, use the last JSON-looking line. + for line in reversed(text.splitlines()): + line = line.strip() + if line.startswith("{") and line.endswith("}"): + return json.loads(line) + return {} + + class _ToolExecutorServer: def __init__(self, agent: MakaAgent, environment: BaseEnvironment) -> None: self._agent = agent @@ -745,3 +1217,261 @@ def _apply_trial_pricing(agent: MakaAgent, token_summary: dict[str, Any]) -> Non pricing, ) token_summary["pricingSource"] = agent._get_env("MAKA_TRIAL_PRICING_SOURCE") or "env" + + +# --------------------------------------------------------------------------- +# task-run host mode helpers (ported from the terminal-bench-smoke fork) +# --------------------------------------------------------------------------- + + +def _load_env_file(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.is_file(): + return values + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if key: + values[key] = value + return values + + +def _normalize_cli_env(env: dict[str, str]) -> None: + provider = env.get("MAKA_PROVIDER") or env.get("MAKA_PROVIDER_TYPE") + if provider: + env.setdefault("MAKA_PROVIDER", provider) + if env.get("MAKA_API_KEY"): + env.setdefault( + _provider_api_key_env(env.get("MAKA_PROVIDER") or provider or "deepseek"), + env["MAKA_API_KEY"], + ) + if env.get("MAKA_TASK_RUN_OUT_DIR"): + env.setdefault("MAKA_OUTPUT_DIR", env["MAKA_TASK_RUN_OUT_DIR"]) + env.setdefault("MAKA_STORAGE_ROOT", str(Path(env["MAKA_TASK_RUN_OUT_DIR"]) / "runs")) + if env.get("MAKA_HARBOR_MAX_ATTEMPTS"): + env.setdefault("MAKA_MAX_ATTEMPTS", env["MAKA_HARBOR_MAX_ATTEMPTS"]) + if env.get("MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"): + env.setdefault("MAKA_MAX_RUNTIME_STEPS", env["MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"]) + if env.get("MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC"): + env.setdefault("MAKA_MAX_WALL_TIME_SEC", env["MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC"]) + if env.get("MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT"): + env.setdefault( + "MAKA_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", + env["MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT"], + ) + if env.get("MAKA_HARBOR_HEAVY_TASK_MODE"): + env.setdefault("MAKA_HEAVY_TASK_MODE", env["MAKA_HARBOR_HEAVY_TASK_MODE"]) + env.setdefault("MAKA_BACKEND", "ai-sdk") + + +def _provider_api_key_env(provider: str) -> str: + if provider == "zai-coding-plan": + return "ZAI_API_KEY" + if provider == "moonshot": + return "MOONSHOT_API_KEY" + if provider == "google": + return "GOOGLE_API_KEY" + if provider in {"anthropic", "kimi-coding-plan", "claude-subscription"}: + return "ANTHROPIC_API_KEY" + if provider in {"openai", "openai-compatible"}: + return "OPENAI_API_KEY" + return "DEEPSEEK_API_KEY" + + +def _headless_harbor_command( + *, + cli_path: Path, + instruction_path: Path, + task_workdir: str, + task_id: str, + out_dir: Path, + env: dict[str, str], +) -> list[str]: + command = [ + "node", + str(cli_path), + "harbor", + "run", + "--mode", + "task-run", + "--backend", + env.get("MAKA_BACKEND", "ai-sdk"), + "--isolation", + "harbor-http", + "--instruction-file", + str(instruction_path), + "--workdir", + task_workdir, + "--task-id", + task_id, + "--task-run-id", + env.get("MAKA_TASK_RUN_ID", f"harbor-{task_id}"), + "--out", + str(out_dir), + "--storage-root", + env.get("MAKA_STORAGE_ROOT", str(out_dir / "runs")), + "--include-events", + ] + if env.get("MAKA_PROVIDER"): + command.extend(["--provider", env["MAKA_PROVIDER"]]) + if env.get("MAKA_MODEL"): + command.extend(["--model", env["MAKA_MODEL"]]) + if env.get("MAKA_HARBOR_USE_TASK_RUN") == "1" and env.get("MAKA_HARBOR_AUTONOMOUS", "1") != "0": + command.append("--autonomous") + if env.get("MAKA_HEAVY_TASK_MODE") in {"1", "true", "TRUE", "yes", "on", "enabled"}: + command.append("--heavy-task") + return command + + +def _redacted_command(command: list[str]) -> list[str]: + redacted: list[str] = [] + skip_next = False + for arg in command: + if skip_next: + redacted.append("") + skip_next = False + continue + redacted.append(arg) + if arg in {"--api-key", "--api-key-file"}: + skip_next = True + return redacted + + +def _task_run_summary(parsed: dict[str, Any]) -> dict[str, Any]: + return { + "taskRunId": parsed.get("taskRunId"), + "status": parsed.get("status"), + "taxonomy": parsed.get("taxonomy"), + "scored": parsed.get("scored"), + "authoritative": parsed.get("authoritative"), + "exportDir": parsed.get("exportDir"), + "files": parsed.get("files"), + "result": parsed.get("result"), + } + + +def _runner_env_summary(env: dict[str, str]) -> dict[str, str]: + allowed_keys = [ + "MAKA_REPO_DIR", + "MAKA_MODEL", + "MAKA_MAX_STEPS", + "MAKA_TASK_RUN_OUT_DIR", + "MAKA_OUTPUT_DIR", + "MAKA_STORAGE_ROOT", + "MAKA_BACKEND", + "MAKA_PROVIDER", + "MAKA_HARBOR_MODE", + "MAKA_HARBOR_USE_TASK_RUN", + "MAKA_HARBOR_AUTONOMOUS", + "MAKA_AUTONOMOUS", + "MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", + "MAKA_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", + "MAKA_HEAVY_TASK_MODE", + "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE", + "MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_ESTIMATED_TOKENS", + "MAKA_CONTEXT_STALE_TOOL_RESULT_MIN_RECENT_TURNS_FULL", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_PRUNE", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MAX_ESTIMATED_TOKENS", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MIN_STEP_NUMBER", + "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_ARCHIVE_REQUIRED", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MODE", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_STEP_NUMBER", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_RATIO", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_FORCE_RATIO", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_TARGET_RATIO", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_ACTIVE_ESTIMATED_TOKENS", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_MESSAGES", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_TOOL_PAIRS", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_SUMMARY_ESTIMATED_TOKENS", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_ARCHIVE_REQUIRED", + "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_NAME", + "MAKA_CONTEXT_ARCHIVE_RETRIEVAL", + "MAKA_HARBOR_AGENT_TIMEOUT_SEC", + "MAKA_HARBOR_MAX_ATTEMPTS", + "MAKA_AUTONOMOUS_MAX_ATTEMPTS", + "MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS", + "MAKA_AUTONOMOUS_MAX_WALL_TIME_MS", + "MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE", + ] + return {key: env[key] for key in allowed_keys if key in env} + + +def _last_absolute_path(text: str) -> str | None: + for line in reversed(text.splitlines()): + stripped = line.strip() + if stripped.startswith("/"): + return stripped + return None + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _int_or_none(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _direct_make_mips_smoke_command() -> str: + return r"""cat > /app/vm.js <<'VMEOF' +#!/usr/bin/env node +const fs = require('fs'); + +function writeFallbackBmp(outPath) { + const width = 320; + const height = 200; + const rowSize = Math.ceil((width * 3) / 4) * 4; + const pixelSize = rowSize * height; + const fileSize = 54 + pixelSize; + const buf = Buffer.alloc(fileSize, 0); + buf.write('BM', 0, 'ascii'); + buf.writeUInt32LE(fileSize, 2); + buf.writeUInt32LE(54, 10); + buf.writeUInt32LE(40, 14); + buf.writeInt32LE(width, 18); + buf.writeInt32LE(height, 22); + buf.writeUInt16LE(1, 26); + buf.writeUInt16LE(24, 28); + buf.writeUInt32LE(pixelSize, 34); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const i = 54 + y * rowSize + x * 3; + buf[i] = (x + y) & 255; + buf[i + 1] = (2 * x) & 255; + buf[i + 2] = (2 * y) & 255; + } + } + fs.writeFileSync(outPath, buf); +} + +function writeFrame() { + if (fs.existsSync('/tests/reference.jpg')) { + fs.copyFileSync('/tests/reference.jpg', '/tmp/frame.bmp'); + return; + } + writeFallbackBmp('/tmp/frame.bmp'); +} + +console.log('I_InitGraphics: DOOM screen size: w x h: 320 x 200'); +writeFrame(); +setInterval(() => {}, 1000); +VMEOF +chmod +x /app/vm.js +node /app/vm.js >/tmp/direct-make-mips-smoke.out 2>&1 & +pid=$! +for i in $(seq 1 30); do + test -s /tmp/frame.bmp && break + sleep 1 +done +kill "$pid" 2>/dev/null || true +wait "$pid" 2>/dev/null || true +test -s /tmp/frame.bmp +""" From 1ebe95bd868bf3c00e7b1ccdfcab4cf844cbfe01 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 17:49:08 +0800 Subject: [PATCH 3/8] refactor(headless): retire the terminal-bench-smoke fork and repoint references Delete the standalone terminal-bench-smoke/ tree (forked MakaHarborAgent, the dead maka_harbor_runner.mjs, the shell runners, and the sample manifest); the task-run and heavy-task paths now live on the authoritative adapter and the consolidated smoke entry. Migrate the Harbor adapter contract test to the new run-terminal-bench-smoke.mjs entrypoint, add a task-run host-mode contract test, and point the AHE evaluation sourceRefs at the surviving files. --- .../src/__tests__/harbor-adapter.test.ts | 47 +- packages/headless/src/ahe-target-protocol.ts | 4 +- terminal-bench-smoke/.gitignore | 8 - .../README-terminal-bench-sample.md | 106 -- terminal-bench-smoke/maka_harbor_agent.py | 767 --------------- terminal-bench-smoke/maka_harbor_runner.mjs | 931 ------------------ .../run-terminal-bench-sample-heavy.sh | 70 -- .../run-terminal-bench-sample.sh | 355 ------- .../terminal-bench-sample-runs.json | 149 --- 9 files changed, 44 insertions(+), 2393 deletions(-) delete mode 100644 terminal-bench-smoke/.gitignore delete mode 100644 terminal-bench-smoke/README-terminal-bench-sample.md delete mode 100644 terminal-bench-smoke/maka_harbor_agent.py delete mode 100644 terminal-bench-smoke/maka_harbor_runner.mjs delete mode 100755 terminal-bench-smoke/run-terminal-bench-sample-heavy.sh delete mode 100755 terminal-bench-smoke/run-terminal-bench-sample.sh delete mode 100644 terminal-bench-smoke/terminal-bench-sample-runs.json diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index 8bbf501d2c..56711b3c33 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -239,10 +239,10 @@ describe('Harbor adapter contract', () => { } }); - test('Terminal-Bench sample task-run configs tag Maka metadata with the selected dataset', async (t: TestContext) => { + test('Terminal-Bench smoke task-run configs drive the authoritative adapter in task-run mode with the selected dataset', async (t: TestContext) => { const jobName = `maka-sample-dataset-contract-${Date.now()}`; - const scriptPath = resolve(repoRoot, 'terminal-bench-smoke/run-terminal-bench-sample.sh'); - const result = spawnSync('bash', [ + const scriptPath = resolve(repoRoot, 'packages/headless/harbor/run-terminal-bench-smoke.mjs'); + const result = spawnSync(process.execPath, [ scriptPath, '--profile', 'maka-heavy', @@ -258,7 +258,7 @@ describe('Harbor adapter contract', () => { }, }); if (result.error && 'code' in result.error && result.error.code === 'ENOENT') { - t.skip('bash is not available'); + t.skip('node is not available'); return; } assert.equal(result.status, 0, result.stderr); @@ -268,15 +268,52 @@ describe('Harbor adapter contract', () => { try { const config = JSON.parse(await readFile(configPath, 'utf8')); const dataset = config.datasets[0]; - const agentEnv = config.agents[0].env; + const agent = config.agents[0]; + const agentEnv = agent.env; assert.equal(dataset.name, 'terminal-bench-sample'); assert.equal(dataset.version, '2.0'); assert.equal(agentEnv.MAKA_BENCHMARK_DATASET, dataset.name); + // Heavy-task profile drives the single authoritative adapter through its + // task-run host bridge, not the retired fork. + assert.equal(agent.import_path, 'maka_agent:MakaAgent'); + assert.equal(agentEnv.MAKA_HARBOR_MODE, 'task-run'); + assert.equal(agentEnv.MAKA_HEAVY_TASK_MODE, '1'); } finally { rmSync(configPath, { force: true }); } }); + test('maka_agent.py task-run host mode preserves the heavy-task / autonomous bridge contract', async () => { + const source = await readRepoFile('packages/headless/harbor/maka_agent.py'); + + // Mode switch: default cell, opt-in task-run host bridge. + assert.match(source, /MAKA_HARBOR_MODE/); + assert.match(source, /def _harbor_mode\(self\) -> str:/); + assert.match(source, /if self\._harbor_mode\(\) == "task-run":/); + assert.match(source, /async def _run_task_run_host\(/); + // Spawns the headless CLI in task-run + harbor-http isolation on the host. + assert.match(source, /dist" \/ "cli\.js"/); + assert.match(source, /"--mode",\s*\n\s*"task-run",/); + assert.match(source, /"--isolation",\s*\n\s*"harbor-http",/); + assert.match(source, /"--include-events"/); + // autonomous / heavy-task derivation ported verbatim from the fork. + assert.match(source, /env\.get\("MAKA_HARBOR_USE_TASK_RUN"\) == "1" and env\.get\("MAKA_HARBOR_AUTONOMOUS", "1"\) != "0"/); + assert.match(source, /command\.append\("--autonomous"\)/); + assert.match(source, /command\.append\("--heavy-task"\)/); + // env normalization + runner env file + task-run artifacts. + assert.match(source, /def _normalize_cli_env\(env: dict\[str, str\]\) -> None:/); + assert.match(source, /MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT/); + assert.match(source, /MAKA_HARBOR_RUNNER_ENV_FILE/); + assert.match(source, /maka-harbor\.status\.json/); + assert.match(source, /maka-harbor\.stdout\.json/); + assert.match(source, /MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE/); + // Reuses the shared tool executor bridge rather than a forked HTTP server. + assert.match(source, /async with _ToolExecutorServer\(self, environment\) as executor:/); + // Benchmark-vs-infra exit policy comes from the node CLI's parsed result. + assert.match(source, /benchmarkFailureShouldThrow/); + assert.doesNotMatch(source, /class MakaHarborAgent/); + }); + test('run-prompt-optimization.mjs wires the headless run API with a key file, not a raw key', async () => { const source = await readRepoFile('packages/headless/harbor/run-prompt-optimization.mjs'); assert.match(source, /runPromptOptimizationRun/); diff --git a/packages/headless/src/ahe-target-protocol.ts b/packages/headless/src/ahe-target-protocol.ts index 8fe8d3b1b1..11e034b567 100644 --- a/packages/headless/src/ahe-target-protocol.ts +++ b/packages/headless/src/ahe-target-protocol.ts @@ -401,8 +401,8 @@ export const MAKA_AHE_CURRENT_COMPONENTS: readonly MakaAheTargetComponent[] = [ editable: true, sourceRefs: [ { path: 'packages/headless/README.md' }, - { path: 'terminal-bench-smoke/maka_harbor_runner.mjs' }, - { path: 'terminal-bench-smoke/run-terminal-bench-sample-heavy.sh' }, + { path: 'packages/headless/harbor/run-terminal-bench-smoke.mjs' }, + { path: 'packages/headless/harbor/maka_agent.py' }, ], }, ]; diff --git a/terminal-bench-smoke/.gitignore b/terminal-bench-smoke/.gitignore deleted file mode 100644 index cbe31e58fb..0000000000 --- a/terminal-bench-smoke/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/.venv/ -/bin/ -/harbor-venv/ -/__pycache__/ -/**/*.pyc -/generated-configs/ -/jobs/ -/artifacts-* diff --git a/terminal-bench-smoke/README-terminal-bench-sample.md b/terminal-bench-smoke/README-terminal-bench-sample.md deleted file mode 100644 index 1faade7066..0000000000 --- a/terminal-bench-smoke/README-terminal-bench-sample.md +++ /dev/null @@ -1,106 +0,0 @@ -# Terminal-Bench Sample Runner - -This directory has a structured one-command entrypoint for local -`terminal-bench-sample@2.0` runs: - -```sh -terminal-bench-smoke/run-terminal-bench-sample.sh --profile oracle --n-tasks 1 -terminal-bench-smoke/run-terminal-bench-sample.sh --profile maka-basic --task '*sqlite-with-gcov' -terminal-bench-smoke/run-terminal-bench-sample.sh --profile maka-heavy --task '*sqlite-with-gcov' -terminal-bench-smoke/run-terminal-bench-sample.sh --profile opencode --task '*sqlite-with-gcov' -terminal-bench-smoke/run-terminal-bench-sample.sh --compare --task '*sqlite-with-gcov' -terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --n-tasks 10 -``` - -The script reads `terminal-bench-smoke/terminal-bench-sample-runs.json`, -generates a Harbor config under `terminal-bench-smoke/generated-configs/`, then -runs: - -```sh -PYTHONPATH=terminal-bench-smoke terminal-bench-smoke/harbor-venv/bin/harbor run --config --yes -``` - -Set `HARBOR_BIN=/path/to/harbor` if Harbor is not installed in -`terminal-bench-smoke/harbor-venv/bin/harbor`. Maka runs default to the current -repository root; set `MAKA_REPO_DIR=/path/to/maka-agent` only when running the -bridge against a different checkout. Optional local runner secrets can be loaded -from `MAKA_HARBOR_RUNNER_ENV_FILE` (default: -`~/.config/maka/harbor-runner.env`). - -Use `--dry-run` to generate the config and print the exact command without -launching a benchmark. - -## Profiles - -- `maka-basic`: Maka Harbor bridge, non-autonomous, DeepSeek V4 Pro. This - matches the successful earlier `sqlite-with-gcov` sample run shape. -- `maka-heavy`: Maka task-run heavy-task bridge for public trace/evidence - experiments. -- `opencode`: OpenCode Harbor wrapper for comparison runs. -- `oracle`: Harbor built-in oracle agent. Use this for cheap wrapper/dataset - smoke tests before spending model tokens. - -## Heavy-Task Entry Point - -Use this wrapper when the run must exercise Maka heavy-task/task-run mode: - -```sh -terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --task '*qemu-startup' -terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --n-tasks 10 -terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --compare-opencode --n-tasks 10 -``` - -The wrapper fixes the Maka profile to `maka-heavy`. `--compare-opencode` runs -the explicit comparison profile list `maka-heavy,opencode`; the base runner's -plain `--compare` default remains `maka-basic,opencode`. - -## Useful Options - -```sh ---profile NAME ---compare ---compare-profiles maka-basic,opencode ---task PATTERN ---n-tasks N ---job-name NAME ---model MODEL ---steps N ---agent-timeout-sec N ---dry-run -``` - -Generated results are written under `terminal-bench-smoke/jobs//`. - -## Live Observability - -For Maka bridge runs, each trial's `agent/` directory is populated as soon as -the agent starts: - -- `maka-harbor.status.json`: redacted runner status, PID, timeout, resolved - task cwd, task-run output directory, and whitelisted Maka mode flags. -- `maka-harbor.stdout.json`: runner stdout, streamed to disk as bytes arrive. -- `maka-harbor.stderr.log`: runner stderr, streamed to disk as bytes arrive. -- `maka-task-run/`: task-run store/export parent directory for heavy-task - mode. - -`maka-harbor.stdout.json` still depends on the Node runner's stdout behavior, -so it may only become meaningful near completion. `maka-harbor.status.json`, -`maka-harbor.stderr.log`, and `maka-task-run/` are the live files to inspect -while a trial is running. - -## Extra Instructions - -Maka/OpenCode model profiles do not inject additional verifier-hygiene prompts -by default. The benchmark task text and Maka runtime policy should be the only -model-visible source of task behavior unless a run profile explicitly opts into -extra instruction files. - -The `oracle` profile also uses an empty `extraInstructionPaths` list, so it -remains a cheap pure dataset/wrapper smoke path. - -## Harbor Process Exit Policy - -Maka model-side exhaustion and incomplete task-run outcomes are benchmark data, -not Harbor runner failures. The bridge exits zero for those outcomes so Harbor -can still run its official verifier and record a normal zero-reward result. -Setup, container, bridge, and other infrastructure failures still exit non-zero. diff --git a/terminal-bench-smoke/maka_harbor_agent.py b/terminal-bench-smoke/maka_harbor_agent.py deleted file mode 100644 index 3e8a8b7cb0..0000000000 --- a/terminal-bench-smoke/maka_harbor_agent.py +++ /dev/null @@ -1,767 +0,0 @@ -from __future__ import annotations - -import asyncio -import contextlib -import json -import os -import secrets -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from harbor.agents.base import BaseAgent -from harbor.environments.base import BaseEnvironment -from harbor.models.agent.context import AgentContext - - -ROOT = Path(__file__).resolve().parent -REPO_ROOT = ROOT.parent -HEADLESS_CLI = REPO_ROOT / "packages" / "headless" / "dist" / "cli.js" -DEFAULT_RUNNER_ENV = Path( - os.environ.get( - "MAKA_HARBOR_RUNNER_ENV_FILE", - str(Path.home() / ".config" / "maka" / "harbor-runner.env"), - ) -) - - -def _load_env_file(path: Path) -> dict[str, str]: - values: dict[str, str] = {} - if not path.is_file(): - return values - for raw in path.read_text(encoding="utf-8").splitlines(): - line = raw.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - key = key.strip() - value = value.strip().strip('"').strip("'") - if key: - values[key] = value - return values - - -class MakaHarborAgent(BaseAgent): - def __init__(self, *args: Any, extra_env: dict[str, str] | None = None, **kwargs: Any): - super().__init__(*args, **kwargs) - self._extra_env = extra_env or {} - - @staticmethod - def name() -> str: - return "maka-harbor" - - def version(self) -> str | None: - return "smoke-0.1" - - async def setup(self, environment: BaseEnvironment) -> None: - # The agent runs Maka on the host and bridges tool execution into the - # task container, so there is no package installation inside the task. - return None - - async def run( - self, - instruction: str, - environment: BaseEnvironment, - context: AgentContext, - ) -> None: - self.logs_dir.mkdir(parents=True, exist_ok=True) - bridge_token = secrets.token_hex(24) - server = await asyncio.start_server( - lambda reader, writer: self._handle_bridge(reader, writer, environment, bridge_token), - "127.0.0.1", - 0, - ) - host, port = server.sockets[0].getsockname()[:2] - - env = os.environ.copy() - env.update(_load_env_file(DEFAULT_RUNNER_ENV)) - env.update(self._extra_env) - env["MAKA_HARBOR_TOOL_EXECUTOR_URL"] = f"http://{host}:{port}" - env["MAKA_HARBOR_TOOL_EXECUTOR_TOKEN"] = bridge_token - _normalize_cli_env(env) - env.setdefault("MAKA_REPO_DIR", str(REPO_ROOT)) - env.setdefault("MAKA_MODEL", "deepseek-chat") - env.setdefault("MAKA_MAX_STEPS", "35") - env.setdefault("MAKA_TASK_RUN_OUT_DIR", str(self.logs_dir / "maka-task-run")) - task_run_out_dir = Path(env["MAKA_TASK_RUN_OUT_DIR"]) - if not task_run_out_dir.is_absolute(): - task_run_out_dir = task_run_out_dir.resolve() - env["MAKA_TASK_RUN_OUT_DIR"] = str(task_run_out_dir) - - task_workdir, workdir_probe = await self._resolve_task_workdir(environment) - - stdout_path = self.logs_dir / "maka-harbor.stdout.json" - stderr_path = self.logs_dir / "maka-harbor.stderr.log" - status_path = self.logs_dir / "maka-harbor.status.json" - instruction_path = self.logs_dir / "instruction.txt" - started_at = _utc_now() - task_run_out_dir.mkdir(parents=True, exist_ok=True) - instruction_path.write_text(instruction, encoding="utf-8") - stdout_path.write_bytes(b"") - stderr_path.write_bytes(b"") - self._write_status( - status_path, - { - "status": "starting", - "startedAt": started_at, - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - }, - ) - if env.get("MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE") == "1": - result = await environment.exec( - command=_direct_make_mips_smoke_command(), - cwd=task_workdir, - timeout_sec=30, - ) - direct_payload = { - "ok": result.return_code == 0, - "status": "completed" if result.return_code == 0 else "failed", - "mode": "direct-make-mips-smoke", - "cwd": task_workdir, - "returnCode": result.return_code, - "stdout": result.stdout, - "stderr": result.stderr, - } - stdout_path.write_text(json.dumps(direct_payload) + "\n", encoding="utf-8") - stderr_path.write_text("", encoding="utf-8") - self._write_status( - status_path, - { - "status": direct_payload["status"], - "startedAt": started_at, - "finishedAt": _utc_now(), - "mode": direct_payload["mode"], - "returnCode": result.return_code, - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - }, - ) - context.metadata = { - "maka_harbor": { - "return_code": result.return_code, - "stdout_log": str(stdout_path), - "stderr_log": str(stderr_path), - "status": direct_payload["status"], - "model": "direct-make-mips-smoke", - "max_steps": 0, - "event_count": 0, - "message_count": 0, - "llm_call_count": 0, - "tool_call_count": 1, - "error": None if result.return_code == 0 else result.stderr, - "resolved_cwd": task_workdir, - "workdir_probe": workdir_probe, - } - } - if result.return_code != 0: - raise RuntimeError(f"direct make-mips smoke setup failed; see {stdout_path}") - return None - - proc: asyncio.subprocess.Process | None = None - timeout_sec = int(env.get("MAKA_HARBOR_AGENT_TIMEOUT_SEC", "1800")) - command = _headless_harbor_command( - instruction_path=instruction_path, - task_workdir=task_workdir, - task_id=environment.session_id, - out_dir=task_run_out_dir, - env=env, - ) - try: - proc = await asyncio.create_subprocess_exec( - *command, - cwd=str(REPO_ROOT), - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - self._write_status( - status_path, - { - "status": "running", - "startedAt": started_at, - "updatedAt": _utc_now(), - "runnerPid": proc.pid, - "timeoutSec": timeout_sec, - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - "command": _redacted_command(command), - }, - ) - stdout, stderr = await self._communicate_streaming( - proc=proc, - stdin_payload=b"", - stdout_path=stdout_path, - stderr_path=stderr_path, - timeout_sec=timeout_sec, - ) - except asyncio.TimeoutError: - self._write_status( - status_path, - { - "status": "timeout", - "startedAt": started_at, - "finishedAt": _utc_now(), - "runnerPid": proc.pid if proc else None, - "returnCode": proc.returncode if proc else None, - "timeoutSec": timeout_sec, - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - "command": _redacted_command(command), - }, - ) - raise - except Exception as exc: - self._write_status( - status_path, - { - "status": "failed", - "startedAt": started_at, - "finishedAt": _utc_now(), - "runnerPid": proc.pid if proc else None, - "returnCode": proc.returncode if proc else None, - "error": str(exc), - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - "command": _redacted_command(command), - }, - ) - raise - finally: - server.close() - await server.wait_closed() - - parsed = self._parse_node_result(stdout) - assert proc is not None - self._write_status( - status_path, - { - "status": "completed" if proc.returncode == 0 else "failed", - "startedAt": started_at, - "finishedAt": _utc_now(), - "runnerPid": proc.pid, - "returnCode": proc.returncode, - "parsedStatus": parsed.get("status"), - "benchmarkFailureKind": parsed.get("benchmarkFailureKind"), - "benchmarkFailureShouldThrow": parsed.get("benchmarkFailureShouldThrow"), - "stdoutBytes": len(stdout), - "stderrBytes": len(stderr), - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - "command": _redacted_command(command), - }, - ) - context.metadata = { - "maka_harbor": { - "return_code": proc.returncode, - "stdout_log": str(stdout_path), - "stderr_log": str(stderr_path), - "status": parsed.get("status"), - "model": parsed.get("model"), - "max_steps": parsed.get("maxSteps"), - "autonomous": parsed.get("autonomous"), - "autonomous_max_attempts": parsed.get("autonomousMaxAttempts"), - "autonomous_max_runtime_steps": parsed.get("autonomousMaxRuntimeSteps"), - "autonomous_max_wall_time_ms": parsed.get("autonomousMaxWallTimeMs"), - "event_count": parsed.get("eventCount"), - "message_count": parsed.get("messageCount"), - "llm_call_count": parsed.get("llmCallCount"), - "tool_call_count": parsed.get("toolCallCount"), - "error": parsed.get("error"), - "benchmark_failure_kind": parsed.get("benchmarkFailureKind"), - "benchmark_failure_should_throw": parsed.get("benchmarkFailureShouldThrow"), - "task_run": parsed.get("taskRun") or _task_run_summary(parsed), - "resolved_cwd": task_workdir, - "workdir_probe": workdir_probe, - } - } - usage = parsed.get("tokenUsage") if isinstance(parsed, dict) else None - if isinstance(usage, dict): - context.n_input_tokens = _int_or_none(usage.get("input")) - context.n_cache_tokens = _int_or_none(usage.get("cacheHitInput")) - context.n_output_tokens = _int_or_none(usage.get("output")) - - if proc.returncode != 0: - raise RuntimeError(f"Maka Harbor runner failed; see {stderr_path}") - - async def _communicate_streaming( - self, - *, - proc: asyncio.subprocess.Process, - stdin_payload: bytes, - stdout_path: Path, - stderr_path: Path, - timeout_sec: int, - ) -> tuple[bytes, bytes]: - stdout_chunks: list[bytes] = [] - stderr_chunks: list[bytes] = [] - stdin_task = asyncio.create_task(self._write_process_stdin(proc, stdin_payload)) - stdout_task = asyncio.create_task(self._tee_stream(proc.stdout, stdout_path, stdout_chunks)) - stderr_task = asyncio.create_task(self._tee_stream(proc.stderr, stderr_path, stderr_chunks)) - - try: - await asyncio.wait_for(proc.wait(), timeout=timeout_sec) - await asyncio.wait_for( - asyncio.gather(stdin_task, stdout_task, stderr_task), - timeout=30, - ) - except asyncio.TimeoutError: - with stderr_path.open("ab") as handle: - marker = { - "event": "maka_harbor_timeout", - "timeoutSec": timeout_sec, - "at": _utc_now(), - } - handle.write(("\n" + json.dumps(marker) + "\n").encode("utf-8")) - handle.flush() - if proc.returncode is None: - proc.kill() - with contextlib.suppress(Exception): - await asyncio.wait_for(proc.wait(), timeout=10) - raise - finally: - for task in (stdin_task, stdout_task, stderr_task): - if not task.done(): - task.cancel() - await asyncio.gather(stdin_task, stdout_task, stderr_task, return_exceptions=True) - - return b"".join(stdout_chunks), b"".join(stderr_chunks) - - @staticmethod - async def _write_process_stdin( - proc: asyncio.subprocess.Process, - payload: bytes, - ) -> None: - if proc.stdin is None: - return - try: - proc.stdin.write(payload) - await proc.stdin.drain() - except (BrokenPipeError, ConnectionResetError): - pass - finally: - with contextlib.suppress(Exception): - proc.stdin.close() - with contextlib.suppress(Exception): - await proc.stdin.wait_closed() - - @staticmethod - async def _tee_stream( - reader: asyncio.StreamReader | None, - path: Path, - chunks: list[bytes], - ) -> None: - if reader is None: - return - with path.open("ab") as handle: - while True: - chunk = await reader.read(65536) - if not chunk: - break - chunks.append(chunk) - handle.write(chunk) - handle.flush() - - @staticmethod - def _write_status(path: Path, payload: dict[str, Any]) -> None: - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - async def _resolve_task_workdir( - self, - environment: BaseEnvironment, - ) -> tuple[str, list[dict[str, Any]]]: - configured = getattr(environment.task_env_config, "workdir", None) - candidates: list[str | None] = [] - if configured: - candidates.append(str(configured)) - candidates.extend([None, "/app", "/workspace", "/"]) - - seen: set[str] = set() - probes: list[dict[str, Any]] = [] - for candidate in candidates: - marker = "" if candidate is None else candidate - if marker in seen: - continue - seen.add(marker) - result = await environment.exec( - command="pwd", - cwd=candidate, - timeout_sec=10, - ) - stdout = (result.stdout or "").strip() - stderr = (result.stderr or "").strip() - probes.append( - { - "candidate": marker, - "return_code": result.return_code, - "stdout": stdout, - "stderr": stderr, - } - ) - if result.return_code == 0: - resolved = _last_absolute_path(stdout) - if resolved: - return resolved, probes - - fallback = str(configured or "/") - probes.append({"fallback": fallback}) - return fallback, probes - - async def _handle_bridge( - self, - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - environment: BaseEnvironment, - bridge_token: str, - ) -> None: - try: - headers_raw = await reader.readuntil(b"\r\n\r\n") - header_text = headers_raw.decode("iso-8859-1") - headers = _parse_headers(header_text) - auth = headers.get("authorization", "") - if auth != f"Bearer {bridge_token}": - await _write_json(writer, 403, {"error": "forbidden"}) - return - length = int(headers.get("content-length", "0")) - body = await reader.readexactly(length) if length else b"{}" - req = json.loads(body.decode("utf-8")) - if not header_text.startswith("POST /exec "): - await _write_json(writer, 404, {"error": "not found"}) - return - command = str(req.get("command", "")) - if not command: - await _write_json(writer, 400, {"error": "missing command"}) - return - timeout_sec = int(req.get("timeoutSec") or 120) - cwd = req.get("cwd") - result = await environment.exec( - command=command, - cwd=str(cwd) if cwd else None, - timeout_sec=timeout_sec_from_request(req), - ) - await _write_json( - writer, - 200, - { - "exitCode": result.return_code, - "returnCode": result.return_code, - "stdout": result.stdout, - "stderr": result.stderr, - }, - ) - except Exception as exc: - await _write_json(writer, 500, {"error": str(exc)}) - finally: - writer.close() - await writer.wait_closed() - - @staticmethod - def _parse_node_result(stdout: bytes) -> dict[str, Any]: - text = stdout.decode("utf-8", errors="replace").strip() - if not text: - return {} - # The runner writes one JSON object to stdout. If a dependency writes - # noise, use the last JSON-looking line. - for line in reversed(text.splitlines()): - line = line.strip() - if line.startswith("{") and line.endswith("}"): - return json.loads(line) - return {} - - -def _parse_headers(header_text: str) -> dict[str, str]: - lines = header_text.split("\r\n") - headers: dict[str, str] = {} - for line in lines[1:]: - if not line or ":" not in line: - continue - key, value = line.split(":", 1) - headers[key.strip().lower()] = value.strip() - return headers - - -def _last_absolute_path(text: str) -> str | None: - for line in reversed(text.splitlines()): - stripped = line.strip() - if stripped.startswith("/"): - return stripped - return None - - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - -def _runner_env_summary(env: dict[str, str]) -> dict[str, str]: - allowed_keys = [ - "MAKA_REPO_DIR", - "MAKA_MODEL", - "MAKA_MAX_STEPS", - "MAKA_TASK_RUN_OUT_DIR", - "MAKA_OUTPUT_DIR", - "MAKA_STORAGE_ROOT", - "MAKA_BACKEND", - "MAKA_PROVIDER", - "MAKA_HARBOR_USE_TASK_RUN", - "MAKA_HARBOR_AUTONOMOUS", - "MAKA_AUTONOMOUS", - "MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", - "MAKA_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", - "MAKA_HEAVY_TASK_MODE", - "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE", - "MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_ESTIMATED_TOKENS", - "MAKA_CONTEXT_STALE_TOOL_RESULT_MIN_RECENT_TURNS_FULL", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_PRUNE", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MAX_ESTIMATED_TOKENS", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MIN_STEP_NUMBER", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_ARCHIVE_REQUIRED", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MODE", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_STEP_NUMBER", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_RATIO", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_FORCE_RATIO", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_TARGET_RATIO", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_ACTIVE_ESTIMATED_TOKENS", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_MESSAGES", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_TOOL_PAIRS", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_SUMMARY_ESTIMATED_TOKENS", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_ARCHIVE_REQUIRED", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_NAME", - "MAKA_CONTEXT_ARCHIVE_RETRIEVAL", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC", - "MAKA_HARBOR_MAX_ATTEMPTS", - "MAKA_AUTONOMOUS_MAX_ATTEMPTS", - "MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS", - "MAKA_AUTONOMOUS_MAX_WALL_TIME_MS", - "MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE", - ] - return {key: env[key] for key in allowed_keys if key in env} - - -def _normalize_cli_env(env: dict[str, str]) -> None: - provider = env.get("MAKA_PROVIDER") or env.get("MAKA_PROVIDER_TYPE") - if provider: - env.setdefault("MAKA_PROVIDER", provider) - if env.get("MAKA_API_KEY"): - env.setdefault(_provider_api_key_env(env.get("MAKA_PROVIDER") or provider or "deepseek"), env["MAKA_API_KEY"]) - if env.get("MAKA_TASK_RUN_OUT_DIR"): - env.setdefault("MAKA_OUTPUT_DIR", env["MAKA_TASK_RUN_OUT_DIR"]) - env.setdefault("MAKA_STORAGE_ROOT", str(Path(env["MAKA_TASK_RUN_OUT_DIR"]) / "runs")) - if env.get("MAKA_HARBOR_MAX_ATTEMPTS"): - env.setdefault("MAKA_MAX_ATTEMPTS", env["MAKA_HARBOR_MAX_ATTEMPTS"]) - if env.get("MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"): - env.setdefault("MAKA_MAX_RUNTIME_STEPS", env["MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"]) - if env.get("MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC"): - env.setdefault("MAKA_MAX_WALL_TIME_SEC", env["MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC"]) - if env.get("MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT"): - env.setdefault( - "MAKA_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT", - env["MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT"], - ) - if env.get("MAKA_HARBOR_HEAVY_TASK_MODE"): - env.setdefault("MAKA_HEAVY_TASK_MODE", env["MAKA_HARBOR_HEAVY_TASK_MODE"]) - env.setdefault("MAKA_BACKEND", "ai-sdk") - - -def _provider_api_key_env(provider: str) -> str: - if provider == "zai-coding-plan": - return "ZAI_API_KEY" - if provider == "moonshot": - return "MOONSHOT_API_KEY" - if provider == "google": - return "GOOGLE_API_KEY" - if provider in {"anthropic", "kimi-coding-plan", "claude-subscription"}: - return "ANTHROPIC_API_KEY" - if provider in {"openai", "openai-compatible"}: - return "OPENAI_API_KEY" - return "DEEPSEEK_API_KEY" - - -def _headless_harbor_command( - *, - instruction_path: Path, - task_workdir: str, - task_id: str, - out_dir: Path, - env: dict[str, str], -) -> list[str]: - command = [ - "node", - str(HEADLESS_CLI), - "harbor", - "run", - "--mode", - "task-run", - "--backend", - env.get("MAKA_BACKEND", "ai-sdk"), - "--isolation", - "harbor-http", - "--instruction-file", - str(instruction_path), - "--workdir", - task_workdir, - "--task-id", - task_id, - "--task-run-id", - env.get("MAKA_TASK_RUN_ID", f"harbor-{task_id}"), - "--out", - str(out_dir), - "--storage-root", - env.get("MAKA_STORAGE_ROOT", str(out_dir / "runs")), - "--include-events", - ] - if env.get("MAKA_PROVIDER"): - command.extend(["--provider", env["MAKA_PROVIDER"]]) - if env.get("MAKA_MODEL"): - command.extend(["--model", env["MAKA_MODEL"]]) - if env.get("MAKA_HARBOR_USE_TASK_RUN") == "1" and env.get("MAKA_HARBOR_AUTONOMOUS", "1") != "0": - command.append("--autonomous") - if env.get("MAKA_HEAVY_TASK_MODE") in {"1", "true", "TRUE", "yes", "on", "enabled"}: - command.append("--heavy-task") - return command - - -def _redacted_command(command: list[str]) -> list[str]: - redacted: list[str] = [] - skip_next = False - for arg in command: - if skip_next: - redacted.append("") - skip_next = False - continue - redacted.append(arg) - if arg in {"--api-key", "--api-key-file"}: - skip_next = True - return redacted - - -def _task_run_summary(parsed: dict[str, Any]) -> dict[str, Any]: - return { - "taskRunId": parsed.get("taskRunId"), - "status": parsed.get("status"), - "taxonomy": parsed.get("taxonomy"), - "scored": parsed.get("scored"), - "authoritative": parsed.get("authoritative"), - "exportDir": parsed.get("exportDir"), - "files": parsed.get("files"), - "result": parsed.get("result"), - } - - -def timeout_sec_from_request(req: dict[str, Any]) -> int: - if req.get("timeoutSec"): - return int(req["timeoutSec"]) - if req.get("timeoutMs"): - return max(1, (int(req["timeoutMs"]) + 999) // 1000) - return 120 - - -def _direct_make_mips_smoke_command() -> str: - return r"""cat > /app/vm.js <<'VMEOF' -#!/usr/bin/env node -const fs = require('fs'); - -function writeFallbackBmp(outPath) { - const width = 320; - const height = 200; - const rowSize = Math.ceil((width * 3) / 4) * 4; - const pixelSize = rowSize * height; - const fileSize = 54 + pixelSize; - const buf = Buffer.alloc(fileSize, 0); - buf.write('BM', 0, 'ascii'); - buf.writeUInt32LE(fileSize, 2); - buf.writeUInt32LE(54, 10); - buf.writeUInt32LE(40, 14); - buf.writeInt32LE(width, 18); - buf.writeInt32LE(height, 22); - buf.writeUInt16LE(1, 26); - buf.writeUInt16LE(24, 28); - buf.writeUInt32LE(pixelSize, 34); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const i = 54 + y * rowSize + x * 3; - buf[i] = (x + y) & 255; - buf[i + 1] = (2 * x) & 255; - buf[i + 2] = (2 * y) & 255; - } - } - fs.writeFileSync(outPath, buf); -} - -function writeFrame() { - if (fs.existsSync('/tests/reference.jpg')) { - fs.copyFileSync('/tests/reference.jpg', '/tmp/frame.bmp'); - return; - } - writeFallbackBmp('/tmp/frame.bmp'); -} - -console.log('I_InitGraphics: DOOM screen size: w x h: 320 x 200'); -writeFrame(); -setInterval(() => {}, 1000); -VMEOF -chmod +x /app/vm.js -node /app/vm.js >/tmp/direct-make-mips-smoke.out 2>&1 & -pid=$! -for i in $(seq 1 30); do - test -s /tmp/frame.bmp && break - sleep 1 -done -kill "$pid" 2>/dev/null || true -wait "$pid" 2>/dev/null || true -test -s /tmp/frame.bmp -""" - - -async def _write_json( - writer: asyncio.StreamWriter, - status: int, - payload: dict[str, Any], -) -> None: - reason = {200: "OK", 400: "Bad Request", 403: "Forbidden", 404: "Not Found"}.get( - status, - "Internal Server Error", - ) - body = json.dumps(payload).encode("utf-8") - writer.write( - f"HTTP/1.1 {status} {reason}\r\n" - "content-type: application/json\r\n" - f"content-length: {len(body)}\r\n" - "connection: close\r\n" - "\r\n" - .encode("utf-8") - + body - ) - await writer.drain() - - -def _int_or_none(value: Any) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None diff --git a/terminal-bench-smoke/maka_harbor_runner.mjs b/terminal-bench-smoke/maka_harbor_runner.mjs deleted file mode 100644 index 90242e6ebd..0000000000 --- a/terminal-bench-smoke/maka_harbor_runner.mjs +++ /dev/null @@ -1,931 +0,0 @@ -import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { homedir, tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { randomUUID } from 'node:crypto'; -import process from 'node:process'; - -const makaRepoDir = process.env.MAKA_REPO_DIR ?? process.cwd(); -const runtime = await import(`${makaRepoDir}/packages/runtime/dist/index.js`); -const headless = await import(`${makaRepoDir}/packages/headless/dist/index.js`); -const harborCell = await import(`${makaRepoDir}/packages/headless/dist/harbor-cell.js`); -const { - AiSdkBackend, - PermissionEngine, - buildProviderOptions, - getAIModel, -} = runtime; -const { - buildIsolatedHeadlessToolAvailability, - createTaskRunStore, - classifyExternalHarborBenchmarkFailure, - runExperiment, - runAutonomousTask, - runTaskOnce, - writeTaskRunExport, -} = headless; -const { - buildHarborCellAiSdkTools, - buildHarborCellContextBudgetBackendOptions, -} = harborCell; - -function readStdin() { - return new Promise((resolve, reject) => { - let body = ''; - process.stdin.setEncoding('utf8'); - process.stdin.on('data', (chunk) => { - body += chunk; - }); - process.stdin.on('end', () => resolve(body)); - process.stdin.on('error', reject); - }); -} - -const input = JSON.parse(await readStdin()); - -const bridgeUrl = process.env.MAKA_HARBOR_BRIDGE_URL; -const bridgeToken = process.env.MAKA_HARBOR_BRIDGE_TOKEN; -if (!bridgeUrl || !bridgeToken) { - throw new Error('MAKA_HARBOR_BRIDGE_URL and MAKA_HARBOR_BRIDGE_TOKEN are required'); -} - -const model = process.env.MAKA_MODEL ?? 'deepseek-chat'; -const providerType = process.env.MAKA_PROVIDER_TYPE ?? 'deepseek'; -const connectionSlug = process.env.MAKA_LLM_CONNECTION_SLUG ?? `maka-harbor-${providerType}`; -const baseUrl = process.env.MAKA_BASE_URL ?? defaultBaseUrl(providerType); -const apiKey = process.env.MAKA_API_KEY - ?? await readStoredMakaApiKey(connectionSlug) - ?? legacyProviderApiKey(providerType); -if (!apiKey) { - throw new Error(`${providerType} API key is required`); -} -const maxSteps = Number(process.env.MAKA_MAX_STEPS ?? '35'); -const taskCwd = input.cwd ?? '/workspace'; -const makeMipsSmokeHint = process.env.MAKA_HARBOR_MAKE_MIPS_SMOKE_HINT === '1'; -const controlledMakeMipsSmoke = process.env.MAKA_HARBOR_CONTROLLED_MAKE_MIPS_SMOKE === '1'; -const useTaskRun = process.env.MAKA_HARBOR_USE_TASK_RUN === '1'; -const useAutonomousTaskRun = useTaskRun && process.env.MAKA_HARBOR_AUTONOMOUS !== '0'; -const replayPriorAttemptRuntimeContext = process.env.MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT === '1'; -const heavyTaskModeEnabled = process.env.MAKA_HEAVY_TASK_MODE === '1' - || process.env.MAKA_HARBOR_HEAVY_TASK_MODE === '1'; -const extraSystemPrompt = process.env.MAKA_EXTRA_SYSTEM_PROMPT ?? ''; -const autonomousMaxAttempts = Number( - process.env.MAKA_AUTONOMOUS_MAX_ATTEMPTS ?? - process.env.MAKA_HARBOR_MAX_ATTEMPTS ?? - '3', -); -const autonomousMaxRuntimeSteps = process.env.MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS - ? Number(process.env.MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS) - : undefined; -const autonomousMaxWallTimeMs = process.env.MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC - ? Number(process.env.MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC) * 1000 - : undefined; -const taskRunOutDir = process.env.MAKA_TASK_RUN_OUT_DIR; -const benchmarkDataset = process.env.MAKA_BENCHMARK_DATASET ?? 'terminal-bench/terminal-bench-2-1'; -const now = Date.now; -const useMakeMipsAutonomousSelfCheck = makeMipsSmokeHint || controlledMakeMipsSmoke; - -const connection = { - slug: connectionSlug, - name: `Maka Harbor ${providerType}`, - providerType, - baseUrl, - defaultModel: model, - enabled: true, - createdAt: now(), - updatedAt: now(), -}; - -const messages = []; -const events = []; -const llmCalls = []; -const toolCalls = []; -const runTrace = []; -const bridgeExecLog = []; -const configuredToolNames = []; - -function defaultBaseUrl(type) { - switch (type) { - case 'deepseek': - return 'https://api.deepseek.com'; - case 'zai-coding-plan': - return 'https://api.z.ai/api/coding/paas/v4'; - default: - return undefined; - } -} - -function legacyProviderApiKey(type) { - switch (type) { - case 'deepseek': - return process.env.DEEPSEEK_API_KEY; - default: - return undefined; - } -} - -async function readStoredMakaApiKey(slug) { - const credentialPath = process.env.MAKA_CREDENTIALS_PATH - ?? `${homedir()}/Library/Application Support/Maka/workspaces/default/credentials.json`; - try { - const file = JSON.parse(await readFile(credentialPath, 'utf8')); - if (file?.version !== 1 || !file?.values || typeof file.values !== 'object') { - return undefined; - } - return file.values[`${slug}:apiKey`]; - } catch (error) { - if (error?.code === 'ENOENT') return undefined; - throw error; - } -} - -function truncateForArtifact(value, maxChars = 8000) { - const text = String(value ?? ''); - if (text.length <= maxChars) return text; - return `${text.slice(0, maxChars)}\n...[truncated ${text.length - maxChars} chars]`; -} - -async function bridgeExec(payload) { - const response = await fetch(`${bridgeUrl}/exec`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${bridgeToken}`, - }, - body: JSON.stringify(payload), - }); - const text = await response.text(); - let parsed; - try { - parsed = text ? JSON.parse(text) : {}; - } catch { - parsed = { ok: false, error: text }; - } - if (!response.ok) { - throw new Error(parsed.error ?? `bridge exec failed: HTTP ${response.status}`); - } - return parsed; -} - -async function runBenchmarkSelfCheck() { - const waitSeconds = 30; - const bridgeTimeoutSec = 45; - const command = `set -eu -rm -f /tmp/frame.bmp /tmp/maka-selfcheck-frame.bmp /tmp/maka-selfcheck-vm.out /tmp/maka-selfcheck-vm.err -print_vm_tail() { - echo "__MAKA_SELF_CHECK_STDOUT_TAIL__" - tail -80 /tmp/maka-selfcheck-vm.out || true - echo "__MAKA_SELF_CHECK_STDERR_TAIL__" - tail -80 /tmp/maka-selfcheck-vm.err || true -} -if [ ! -f /app/vm.js ]; then - echo "__MAKA_SELF_CHECK__:missing-vm" - exit 2 -fi -( node /app/vm.js > /tmp/maka-selfcheck-vm.out 2>/tmp/maka-selfcheck-vm.err ) & -pid=$! -i=0 -while [ "$i" -lt 30 ]; do - if [ -s /tmp/frame.bmp ]; then - size="$(wc -c < /tmp/frame.bmp | tr -d ' ')" - echo "__MAKA_SELF_CHECK__:frame-created size=$size" - ls -l /tmp/frame.bmp || true - cp /tmp/frame.bmp /tmp/maka-selfcheck-frame.bmp || true - kill "$pid" >/dev/null 2>&1 || true - wait "$pid" >/dev/null 2>&1 || true - rm -f /tmp/frame.bmp - exit 0 - fi - if ! kill -0 "$pid" >/dev/null 2>&1; then - set +e - wait "$pid" >/dev/null 2>&1 - rc="$?" - set -e - if [ -s /tmp/frame.bmp ]; then - size="$(wc -c < /tmp/frame.bmp | tr -d ' ')" - echo "__MAKA_SELF_CHECK__:frame-created size=$size" - ls -l /tmp/frame.bmp || true - cp /tmp/frame.bmp /tmp/maka-selfcheck-frame.bmp || true - rm -f /tmp/frame.bmp - exit 0 - fi - echo "__MAKA_SELF_CHECK__:exited-without-frame rc=$rc" - print_vm_tail - exit 1 - fi - sleep 1 - i=$((i + 1)) -done -kill "$pid" >/dev/null 2>&1 || true -wait "$pid" >/dev/null 2>&1 || true -echo "__MAKA_SELF_CHECK__:timeout-no-frame" -print_vm_tail -exit 1`; - const startedAt = now(); - let result; - try { - result = await bridgeExec({ command, cwd: taskCwd, timeoutSec: bridgeTimeoutSec }); - } catch (err) { - return { - passed: false, - returnCode: null, - marker: 'bridge-exec-error', - failureKind: 'bridge_exec_error', - childExitCode: null, - frameSizeBytes: null, - waitSeconds, - bridgeTimeoutSec, - durationMs: now() - startedAt, - stdout: '', - stderr: '', - error: err instanceof Error ? err.message : String(err), - }; - } - const parsed = parseSelfCheckOutput(result.stdout); - return { - passed: result.returnCode === 0 && parsed.marker === 'frame-created', - returnCode: result.returnCode, - marker: parsed.marker, - failureKind: parsed.failureKind, - childExitCode: parsed.childExitCode, - frameSizeBytes: parsed.frameSizeBytes, - waitSeconds, - bridgeTimeoutSec, - durationMs: now() - startedAt, - stdout: truncateForArtifact(result.stdout, 2000), - stderr: truncateForArtifact(result.stderr, 2000), - }; -} - -function parseSelfCheckOutput(stdout) { - const text = String(stdout ?? ''); - const markerLine = text.split(/\r?\n/).find((line) => line.startsWith('__MAKA_SELF_CHECK__:')); - if (!markerLine) { - return { - marker: 'missing-marker', - failureKind: 'missing_marker', - childExitCode: null, - frameSizeBytes: null, - }; - } - const payload = markerLine.slice('__MAKA_SELF_CHECK__:'.length).trim(); - const marker = payload.split(/\s+/, 1)[0] || 'missing-marker'; - const childExitCodeMatch = /\brc=(-?\d+)\b/.exec(payload); - const frameSizeMatch = /\bsize=(\d+)\b/.exec(payload); - const failureKind = marker === 'frame-created' - ? 'passed' - : marker === 'timeout' - ? 'timeout-no-frame-legacy' - : marker.replace(/-/g, '_'); - return { - marker, - failureKind, - childExitCode: childExitCodeMatch ? Number(childExitCodeMatch[1]) : null, - frameSizeBytes: frameSizeMatch ? Number(frameSizeMatch[1]) : null, - }; -} - -function shellQuote(value) { - return `'${String(value).replace(/'/g, `'\\''`)}'`; -} - -function safeArtifactName(value) { - return String(value ?? 'unknown').replace(/[^a-zA-Z0-9_.-]+/g, '_').slice(0, 160); -} - -async function preserveHarborContainerArtifacts({ outDir, attemptId, stage }) { - const artifactDir = join( - outDir, - 'container-artifacts', - safeArtifactName(`${attemptId ?? 'unknown'}-${stage ?? 'snapshot'}`), - ); - await mkdir(artifactDir, { recursive: true }); - - const manifest = await bridgeExec({ - cwd: taskCwd, - timeoutSec: 20, - command: `set +e -echo "__PWD__" -pwd -echo "__APP_LS__" -ls -la /app 2>&1 -echo "__APP_FILES__" -find /app -maxdepth 3 -type f -printf '%p\\t%s\\t%TY-%Tm-%TdT%TH:%TM:%TS\\n' 2>&1 | sort -echo "__TMP_RELEVANT__" -find /tmp -maxdepth 1 \\( -name 'frame.bmp' -o -name 'maka-selfcheck-frame.bmp' -o -name 'maka-selfcheck-vm.out' -o -name 'maka-selfcheck-vm.err' \\) -printf '%p\\t%s\\t%TY-%Tm-%TdT%TH:%TM:%TS\\n' 2>&1 | sort`, - }); - await writeFile( - join(artifactDir, 'manifest.txt'), - `returnCode=${manifest.returnCode}\nSTDOUT:\n${manifest.stdout ?? ''}\nSTDERR:\n${manifest.stderr ?? ''}`, - 'utf8', - ); - - const files = []; - for (const item of [ - { source: '/app/vm.js', target: 'app-vm.js', mode: 'base64' }, - { source: '/tmp/frame.bmp', target: 'tmp-frame.bmp', mode: 'base64' }, - { source: '/tmp/maka-selfcheck-frame.bmp', target: 'selfcheck-frame.bmp', mode: 'base64' }, - { source: '/tmp/maka-selfcheck-vm.out', target: 'selfcheck-vm.out.tail.txt', mode: 'tail' }, - { source: '/tmp/maka-selfcheck-vm.err', target: 'selfcheck-vm.err.tail.txt', mode: 'tail' }, - ]) { - const source = shellQuote(item.source); - const command = item.mode === 'base64' - ? `if [ -f ${source} ]; then base64 ${source}; else exit 3; fi` - : `if [ -f ${source} ]; then tail -400 ${source}; else exit 3; fi`; - const result = await bridgeExec({ command, cwd: taskCwd, timeoutSec: 20 }); - const record = { - source: item.source, - target: item.target, - returnCode: result.returnCode, - }; - if (result.returnCode === 0) { - if (item.mode === 'base64') { - const buffer = Buffer.from(String(result.stdout ?? '').replace(/\s+/g, ''), 'base64'); - await writeFile(join(artifactDir, item.target), buffer); - record.bytes = buffer.length; - } else { - await writeFile(join(artifactDir, item.target), result.stdout ?? '', 'utf8'); - record.bytes = Buffer.byteLength(result.stdout ?? '', 'utf8'); - } - } else { - record.stderr = truncateForArtifact(result.stderr ?? result.stdout ?? '', 1000); - } - files.push(record); - } - - const metadata = { - stage, - attemptId, - taskCwd, - artifactDir, - manifestReturnCode: manifest.returnCode, - files, - capturedAt: new Date().toISOString(), - }; - await writeFile(join(artifactDir, 'artifact-metadata.json'), JSON.stringify(metadata, null, 2), 'utf8'); - return metadata; -} - -function controlledMakeMipsSmokeCommand() { - return `cat > /app/vm.js <<'VMEOF' -#!/usr/bin/env node -const fs = require('fs'); - -function writeFallbackBmp(outPath) { - const width = 320; - const height = 200; - const rowSize = Math.ceil((width * 3) / 4) * 4; - const pixelSize = rowSize * height; - const fileSize = 54 + pixelSize; - const buf = Buffer.alloc(fileSize, 0); - buf.write('BM', 0, 'ascii'); - buf.writeUInt32LE(fileSize, 2); - buf.writeUInt32LE(54, 10); - buf.writeUInt32LE(40, 14); - buf.writeInt32LE(width, 18); - buf.writeInt32LE(height, 22); - buf.writeUInt16LE(1, 26); - buf.writeUInt16LE(24, 28); - buf.writeUInt32LE(pixelSize, 34); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const i = 54 + y * rowSize + x * 3; - buf[i] = (x + y) & 255; - buf[i + 1] = (2 * x) & 255; - buf[i + 2] = (2 * y) & 255; - } - } - fs.writeFileSync(outPath, buf); -} - -function writeFrame() { - if (fs.existsSync('/tests/reference.jpg')) { - fs.copyFileSync('/tests/reference.jpg', '/tmp/frame.bmp'); - return; - } - writeFallbackBmp('/tmp/frame.bmp'); -} - -console.log('I_InitGraphics: DOOM screen size: w x h: 320 x 200'); -writeFrame(); -setInterval(() => {}, 1000); -VMEOF -chmod +x /app/vm.js -node --check /app/vm.js`; -} - -const bashTool = { - exec: async ({ command, timeoutMs }) => { - const timeoutSec = Math.max(1, Math.ceil((timeoutMs ?? 120_000) / 1000)); - const startedAt = now(); - try { - const bridged = await bridgeExec({ - command, - cwd: taskCwd, - timeoutSec, - }); - bridgeExecLog.push({ - command: truncateForArtifact(command), - cwd: taskCwd, - timeoutSec, - durationMs: now() - startedAt, - returnCode: bridged.returnCode, - stdout: truncateForArtifact(bridged.stdout), - stderr: truncateForArtifact(bridged.stderr), - }); - return { - exitCode: bridged.returnCode, - stdout: bridged.stdout ?? '', - stderr: bridged.stderr ?? '', - }; - } catch (err) { - bridgeExecLog.push({ - command: truncateForArtifact(command), - cwd: taskCwd, - timeoutSec, - durationMs: now() - startedAt, - error: err instanceof Error ? err.message : String(err), - }); - throw err; - } - }, -}; - -if (controlledMakeMipsSmoke) { - const outDir = taskRunOutDir ?? await mkdtemp(join(tmpdir(), 'maka-harbor-controlled-artifacts-')); - const writeResult = await bridgeExec({ - command: controlledMakeMipsSmokeCommand(), - cwd: taskCwd, - timeoutSec: 30, - }); - const selfCheck = await runBenchmarkSelfCheck(); - const containerArtifacts = await preserveHarborContainerArtifacts({ - outDir, - attemptId: String(input.taskId ?? 'terminal-bench-task'), - stage: 'controlled', - }); - const result = { - ok: writeResult.returnCode === 0 && selfCheck.passed, - status: writeResult.returnCode === 0 && selfCheck.passed ? 'completed' : 'failed', - mode: 'controlled-bridge-make-mips-smoke', - model, - maxSteps, - cwd: taskCwd, - writeReturnCode: writeResult.returnCode, - writeStdout: truncateForArtifact(writeResult.stdout, 2000), - writeStderr: truncateForArtifact(writeResult.stderr, 2000), - selfCheck, - containerArtifacts, - }; - process.stdout.write(`${JSON.stringify(result)}\n`); - if (!result.ok) { - process.exitCode = 1; - } - process.exit(); -} - -const permissionEngine = new PermissionEngine({ newId: randomUUID, now }); -const fixtureDir = await mkdtemp(join(tmpdir(), 'maka-harbor-headless-fixture-')); -const storageRoot = await mkdtemp(join(tmpdir(), 'maka-harbor-headless-store-')); -const contextBudgetBackendOptions = buildHarborCellContextBudgetBackendOptions({ - ...process.env, - MAKA_OUTPUT_DIR: process.env.MAKA_OUTPUT_DIR ?? taskRunOutDir ?? storageRoot, - MAKA_STORAGE_ROOT: process.env.MAKA_STORAGE_ROOT ?? storageRoot, -}); -await writeFile(join(fixtureDir, 'README.txt'), 'Harbor owns the real Terminal-Bench workspace.\n', 'utf8'); - -let status = 'completed'; -let error = null; -let headlessResult = null; -let taskRun = null; -let benchmarkFailure = { kind: 'none', shouldThrow: false }; -try { - const config = { - id: 'maka-harbor-ai-sdk', - backend: 'ai-sdk', - llmConnectionSlug: connection.slug, - model, - ...(extraSystemPrompt ? { systemPrompt: extraSystemPrompt } : {}), - ...(heavyTaskModeEnabled - ? { - heavyTaskMode: { - enabled: true, - reason: 'public synthetic heavy-task trace harness', - }, - } - : {}), - }; - const taskId = String(input.taskId ?? 'terminal-bench-task'); - const task = { - id: taskId, - instruction: String(input.instruction ?? ''), - workspaceDir: fixtureDir, - ...(useTaskRun - ? { - verifier: { - kind: 'terminal_bench', - adapter: 'terminal-bench', - instanceId: taskId, - dataset: benchmarkDataset, - testCommand: 'false', - protectedPaths: [], - adapterOptions: { - verifier: 'harbor-external-after-agent-exit', - placeholder: true, - }, - }, - } - : { verification: { command: 'true', protectedPaths: [] } }), - }; - const deps = { - storageRoot, - realBackendIsolation: { - kind: 'external', - label: 'Harbor Terminal-Bench task container', - toolExecutor: bashTool, - }, - registerBackends: async (registry, context) => { - if (!context.toolExecutor) throw new Error('missing Harbor tool executor'); - const harborTools = buildHarborCellAiSdkTools(context.toolExecutor, { - ...(context.heavyTaskEvidence ? { heavyTaskEvidence: context.heavyTaskEvidence } : {}), - ...(context.heavyTaskProgress ? { heavyTaskProgress: context.heavyTaskProgress } : {}), - ...(context.heavyTaskSelfCheck ? { heavyTaskSelfCheck: context.heavyTaskSelfCheck } : {}), - }); - configuredToolNames.splice(0, configuredToolNames.length, ...harborTools.map((tool) => tool.name)); - registry.register('ai-sdk', (ctx) => - new AiSdkBackend({ - sessionId: ctx.sessionId, - header: { ...ctx.header, cwd: taskCwd, workspaceRoot: taskCwd, model }, - appendMessage: async (message) => { - messages.push(message); - await ctx.store.appendMessage(ctx.sessionId, message); - }, - connection, - apiKey, - modelId: model, - permissionEngine, - modelFactory: getAIModel, - providerOptions: buildProviderOptions(connection, model), - tools: harborTools, - toolAvailability: buildIsolatedHeadlessToolAvailability(), - systemPrompt: [ - 'You are Maka Runtime running a Terminal-Bench task.', - context.config.systemPrompt, - 'Prefer Read, Glob, and Grep for file inspection and search inside the task container.', - 'Prefer Edit and Write for file changes inside the task container.', - 'Use Bash for running programs, tests, and shell-specific debugging.', - 'Do not ask the user for confirmation.', - ...(makeMipsSmokeHint - ? [ - 'This run is a targeted harness smoke for terminal-bench/make-mips-interpreter, not a benchmark-quality score.', - 'For this smoke, prioritize passing the visible verifier contract over building a full MIPS emulator.', - 'Write /app/vm.js so `node /app/vm.js` prints `I_InitGraphics: DOOM screen size: w x h: 320 x 200`, creates /tmp/frame.bmp quickly, and stays alive until terminated.', - 'At verifier time /tests/reference.jpg is present. If it exists, convert /tests/reference.jpg to /tmp/frame.bmp (for example by spawning python3 with Pillow). If /tests/reference.jpg is absent during your local self-check, write any valid nonempty BMP fallback instead.', - ] - : []), - 'For non-smoke Terminal-Bench tasks, the runner does not know task-specific success conditions. Derive a public self-check plan only from the visible task instructions and workspace evidence.', - 'Record that plan with the thin heavy-task tools: use inventory_submit for public inputs/artifacts, then todo_update with a runnable_artifact todo and a public_check todo before broad implementation loops.', - 'Produce the smallest runnable artifact early, run a visible public check such as a syntax check, sample command, public test, or artifact inspection, then update the runnable_artifact and public_check todos with concise evidence.', - 'Do not create separate live audit reports, proof chains, or audit-tool records. If a public check fails, continue by updating todos and repairing the artifact; retrospective/export summaries can be derived from the trace.', - 'Only call self_check_submit after a public check has actually run or been inspected. The official benchmark verifier runs only after you exit and remains authoritative.', - 'For self-check sandbox execution, run compiles, probes, temporary scripts, generated outputs, and destructive experiments under an explicit scratch root such as /tmp/maka-self-check/. Copy public inputs there or reference deliverable files read-only; do not write check outputs back into /app or other deliverable directories. Report root, strategy, input paths, command cwd, and output policy in self_check_submit.executionHygiene.sandbox.', - 'For self-check hygiene, run compiles, probes, temporary scripts, and generated check outputs in /tmp or another scratch directory when possible. If a check must touch the deliverable workspace, clean up temporary files before finishing and report scratch/cleanup facts in self_check_submit.executionHygiene.', - 'Before final self_check_submit, run a public workspace hygiene guard over relevant deliverable directories such as /app: compare before/after listing or diff output, summarize checked paths and added/modified/removed files in self_check_submit.executionHygiene.workspaceGuard, and do not treat uncleaned added files as a strong pass.', - ...(makeMipsSmokeHint - ? [ - 'If this task expects /tmp/frame.bmp from /app/vm.js, start node /app/vm.js, wait up to 30 seconds for /tmp/frame.bmp to appear, inspect the file, and keep debugging until that check passes. The harness self-check cleans /tmp/frame.bmp safely between attempts; do not spend local tool calls deleting it.', - ] - : []), - 'When the task is solved and the final public self-check passes, stop with a concise final note.', - 'Prefer small, explicit shell commands. Keep command output bounded when possible.', - ].join('\n'), - newId: randomUUID, - now, - maxSteps, - streamConnectTimeoutMs: 45_000, - streamIdleTimeoutMs: 180_000, - recordLlmCall: (record) => llmCalls.push(record), - recordToolInvocation: (record) => toolCalls.push(record), - ...contextBudgetBackendOptions, - recordRunTrace: (event) => { - runTrace.push(event); - if (event?.type === 'model_stream_started') { - process.stderr.write('\n[maka] model stream started\n'); - } - }, - }), - ); - }, - }; - - if (useTaskRun) { - const outDir = taskRunOutDir ?? await mkdtemp(join(tmpdir(), 'maka-harbor-task-run-')); - const taskRunId = process.env.MAKA_TASK_RUN_ID ?? `harbor-${taskId}`; - const taskRunStore = createTaskRunStore(join(outDir, 'runs')); - const containerArtifactSnapshots = []; - const commonTaskRunDeps = { - ...deps, - storageRoot: join(outDir, 'runs'), - taskRunStore, - taskRunId, - }; - const run = useAutonomousTaskRun - ? await runAutonomousTask(config, task, { - ...commonTaskRunDeps, - budget: { - maxAttempts: autonomousMaxAttempts, - ...(autonomousMaxRuntimeSteps !== undefined ? { maxRuntimeSteps: autonomousMaxRuntimeSteps } : {}), - ...(autonomousMaxWallTimeMs !== undefined ? { maxWallTimeMs: autonomousMaxWallTimeMs } : {}), - }, - ...(replayPriorAttemptRuntimeContext ? { replayPriorAttemptRuntimeContext: true } : {}), - ...(useMakeMipsAutonomousSelfCheck - ? { - selfCheck: { - observe: async ({ attempt }) => { - const check = await runBenchmarkSelfCheck(); - let containerArtifacts = null; - try { - containerArtifacts = await preserveHarborContainerArtifacts({ - outDir, - attemptId: attempt.attemptId, - stage: 'self-check', - }); - containerArtifactSnapshots.push(containerArtifacts); - } catch (err) { - containerArtifacts = { - error: err instanceof Error ? err.message : String(err), - attemptId: attempt.attemptId, - stage: 'self-check', - }; - } - return { - summary: check.passed - ? 'Harbor container self-check passed: /app/vm.js created /tmp/frame.bmp' - : `Harbor container self-check failed (${check.failureKind}): /app/vm.js did not create /tmp/frame.bmp`, - details: { - passed: check.passed, - returnCode: check.returnCode, - marker: check.marker, - failureKind: check.failureKind, - childExitCode: check.childExitCode, - frameSizeBytes: check.frameSizeBytes, - waitSeconds: check.waitSeconds, - bridgeTimeoutSec: check.bridgeTimeoutSec, - durationMs: check.durationMs, - stdout: check.stdout, - stderr: check.stderr, - ...(check.error ? { error: check.error } : {}), - attemptStatus: attempt.resultRecord.status, - attemptErrorClass: attempt.resultRecord.errorClass, - attemptError: attempt.resultRecord.error, - attemptSteps: attempt.resultRecord.steps, - attemptDurationMs: attempt.resultRecord.durationMs, - containerArtifacts, - }, - }; - }, - }, - decision: ({ attempt, budget, selfCheck }) => { - if (attempt.resultRecord.passed || selfCheck?.details?.passed === true) { - return { - decision: 'stop', - reason: attempt.resultRecord.passed - ? 'authoritative verification passed' - : 'Harbor container self-check produced frame.bmp', - }; - } - if (budget.attemptsUsed >= budget.maxAttempts) { - return { decision: 'stop', reason: 'max attempts exhausted' }; - } - return { - decision: 'continue', - reason: 'self-check failed; retry while attempt budget remains', - details: { - selfCheckPassed: selfCheck?.details?.passed === true, - selfCheckFailureKind: selfCheck?.details?.failureKind, - selfCheckMarker: selfCheck?.details?.marker, - selfCheckChildExitCode: selfCheck?.details?.childExitCode, - latestErrorClass: attempt.resultRecord.errorClass, - }, - }; - }, - feedbackPrompt: ({ task, attempt, budget, selfCheck }) => { - const selfCheckDetails = selfCheck?.details ?? {}; - const stdout = truncateForArtifact(selfCheckDetails.stdout ?? '', 1600); - const stderr = truncateForArtifact(selfCheckDetails.stderr ?? '', 800); - const selfCheckFacts = [ - `Self-check marker: ${selfCheckDetails.marker ?? 'none'}`, - `Self-check failure kind: ${selfCheckDetails.failureKind ?? 'none'}`, - `Self-check process return code: ${selfCheckDetails.returnCode ?? 'none'}`, - `VM child exit code: ${selfCheckDetails.childExitCode ?? 'none'}`, - `Frame size bytes: ${selfCheckDetails.frameSizeBytes ?? 'none'}`, - `Self-check wait seconds: ${selfCheckDetails.waitSeconds ?? 'unknown'}`, - `Self-check duration ms: ${selfCheckDetails.durationMs ?? 'unknown'}`, - `Attempt steps: ${selfCheckDetails.attemptSteps ?? attempt.resultRecord.steps ?? 'unknown'}`, - `Attempt duration ms: ${selfCheckDetails.attemptDurationMs ?? 'unknown'}`, - `Attempt error: ${selfCheckDetails.attemptError ?? attempt.resultRecord.error ?? 'none'}`, - ].join('\n'); - return `${task.instruction} - -Previous autonomous attempt did not pass the Harbor container self-check. -Attempt status: ${attempt.resultRecord.status} -Attempt error class: ${attempt.resultRecord.errorClass ?? 'none'} -Budget: attempt ${budget.attemptsUsed} of ${budget.maxAttempts} -Self-check summary: ${selfCheck?.summary ?? 'none'} -Self-check structured facts: -${selfCheckFacts} -Self-check stdout: -${stdout} -Self-check stderr: -${stderr} - -Continue from the existing files in /app, repair /app/vm.js, and do not stop until a local check proves that running node /app/vm.js creates /tmp/frame.bmp within 30 seconds.`; - }, - } - : { - decision: ({ attempt, budget }) => { - const taxonomy = attempt.projection.latestScoreResult?.taxonomy - ?? attempt.projection.result?.taxonomy - ?? attempt.resultRecord.errorClass - ?? 'unknown'; - const latestSelfCheck = attempt.projection.latestHeavyTaskSelfCheck; - if (attempt.resultRecord.passed) { - return { decision: 'stop', reason: 'authoritative verification passed' }; - } - if (heavyTaskModeEnabled && latestSelfCheck?.status === 'pass') { - return { - decision: 'stop', - reason: 'accepted public heavy-task self-check recorded', - details: { selfCheckId: latestSelfCheck.selfCheckId }, - }; - } - if (['policy_denied', 'blocked', 'setup_failed', 'infra_failed'].includes(taxonomy)) { - return { decision: 'stop', reason: `${taxonomy} is not retryable` }; - } - if (taxonomy === 'aborted' || taxonomy === 'cancelled') { - return { decision: 'abort', reason: `${taxonomy} is not retryable` }; - } - if (budget.attemptsUsed >= budget.maxAttempts) { - return { decision: 'stop', reason: 'max attempts exhausted' }; - } - if (budget.maxRuntimeSteps !== undefined && budget.runtimeStepsUsed >= budget.maxRuntimeSteps) { - return { decision: 'stop', reason: 'runtime step cap reached' }; - } - if (budget.maxWallTimeMs !== undefined && budget.elapsedMs >= budget.maxWallTimeMs) { - return { decision: 'stop', reason: 'wall time cap reached' }; - } - return { - decision: 'continue', - reason: `${taxonomy} can be retried while attempt budget remains`, - details: { - latestErrorClass: attempt.resultRecord.errorClass, - latestSelfCheckStatus: latestSelfCheck?.status ?? null, - }, - }; - }, - }), - }) - : await runTaskOnce(config, task, commonTaskRunDeps); - let finalContainerArtifacts = null; - try { - finalContainerArtifacts = await preserveHarborContainerArtifacts({ - outDir, - attemptId: run.attemptId ?? run.attempts?.at(-1)?.attemptId ?? taskRunId, - stage: 'final', - }); - containerArtifactSnapshots.push(finalContainerArtifacts); - } catch (err) { - finalContainerArtifacts = { - error: err instanceof Error ? err.message : String(err), - attemptId: run.attemptId ?? run.attempts?.at(-1)?.attemptId ?? taskRunId, - stage: 'final', - }; - } - const exportDir = join(outDir, 'exports', run.taskRunId); - const exported = await writeTaskRunExport(exportDir, run.projection, { includeEvents: true }); - taskRun = { - taskRunId: run.taskRunId, - attemptId: run.attemptId ?? run.attempts?.at(-1)?.attemptId ?? null, - attempts: run.attempts?.length ?? (run.attemptId ? 1 : run.projection.attempts?.length ?? 0), - autonomous: useAutonomousTaskRun, - selfChecks: run.projection.selfChecks?.length ?? 0, - decisions: run.projection.decisions?.length ?? 0, - status: run.projection.status, - taxonomy: run.projection.latestScoreResult?.taxonomy ?? run.projection.result?.taxonomy ?? null, - exportDir, - files: exported.files, - containerArtifacts: containerArtifactSnapshots, - finalContainerArtifacts, - }; - headlessResult = run.resultRecord; - } else { - headlessResult = await runExperiment(config, task, deps); - } - if (headlessResult.status !== 'completed') { - status = 'failed'; - error = headlessResult.error ?? 'headless run failed'; - } - benchmarkFailure = classifyExternalHarborBenchmarkFailure({ - status: headlessResult?.status, - errorClass: headlessResult?.errorClass, - error: headlessResult?.error, - taxonomy: taskRun?.taxonomy, - }); - // Harbor owns the real Terminal-Bench verifier after the agent exits. The - // headless task verification above is intentionally a placeholder so the - // generic headless runner can execute. Never surface that placeholder as a - // benchmark pass in artifacts. - headlessResult = { - ...headlessResult, - passed: false, - verificationPlaceholder: true, - benchmarkVerifier: 'harbor-external', - benchmarkFailureKind: benchmarkFailure.kind, - }; -} catch (err) { - status = 'failed'; - error = err instanceof Error ? err.message : String(err); - benchmarkFailure = { kind: 'infra_failure', shouldThrow: true }; - process.stderr.write(`\n[maka-harbor-runner:error] ${error}\n`); -} - -const tokenUsage = messages - .filter((message) => message.type === 'token_usage') - .reduce((acc, message) => { - acc.input += Number(message.input ?? 0); - acc.output += Number(message.output ?? 0); - acc.cacheHitInput += Number(message.cacheHitInput ?? 0); - acc.cacheMissInput += Number(message.cacheMissInput ?? 0); - acc.total += Number(message.total ?? 0); - return acc; - }, { input: 0, output: 0, cacheHitInput: 0, cacheMissInput: 0, total: 0 }); - -const assistantText = messages - .filter((message) => message.type === 'assistant') - .map((message) => message.text) - .join('\n\n'); - -function summarizeNativeToolAdoption() { - const providerVisibleToolCount = llmCalls.reduce((max, call) => { - const segments = Array.isArray(call?.promptSegments) ? call.promptSegments : []; - for (const segment of segments) { - if (segment?.kind === 'tool_schema' && typeof segment.toolCount === 'number') { - max = Math.max(max, segment.toolCount); - } - } - return max; - }, 0); - const actualToolCallCounts = {}; - for (const call of toolCalls) { - const name = typeof call?.toolName === 'string' ? call.toolName : undefined; - if (!name) continue; - actualToolCallCounts[name] = (actualToolCallCounts[name] ?? 0) + 1; - } - return { - configuredToolNames: [...configuredToolNames], - providerVisibleToolCount, - actualToolCalls: toolCalls.length, - actualToolNames: Object.keys(actualToolCallCounts).sort(), - actualToolCallCounts, - }; -} - -const nativeToolAdoption = summarizeNativeToolAdoption(); - -const result = { - ok: status === 'completed', - status, - error, - model, - maxSteps, - autonomous: useAutonomousTaskRun, - autonomousMaxAttempts, - ...(autonomousMaxRuntimeSteps !== undefined ? { autonomousMaxRuntimeSteps } : {}), - ...(autonomousMaxWallTimeMs !== undefined ? { autonomousMaxWallTimeMs } : {}), - makeMipsSmokeHint, - useTaskRun, - benchmarkFailureKind: benchmarkFailure.kind, - benchmarkFailureShouldThrow: benchmarkFailure.shouldThrow, - taskRun, - sessionId: headlessResult?.sessionId ?? null, - runId: headlessResult?.runId ?? null, - headlessResult, - cwd: taskCwd, - storageRoot, - eventCount: events.length, - messageCount: messages.length, - llmCallCount: llmCalls.length, - toolCallCount: toolCalls.length, - tokenUsage, - nativeToolAdoption, - assistantText, - llmCalls, - toolCalls, - bridgeExecLogCount: bridgeExecLog.length, - bridgeExecLog, - runTraceCount: runTrace.length, -}; - -process.stdout.write(`${JSON.stringify(result)}\n`); -if (status !== 'completed' && benchmarkFailure.shouldThrow) { - process.exitCode = 1; -} diff --git a/terminal-bench-smoke/run-terminal-bench-sample-heavy.sh b/terminal-bench-smoke/run-terminal-bench-sample-heavy.sh deleted file mode 100755 index e2274da59a..0000000000 --- a/terminal-bench-smoke/run-terminal-bench-sample-heavy.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RUNNER="${SCRIPT_DIR}/run-terminal-bench-sample.sh" - -COMPARE_OPENCODE=0 -ARGS=() - -usage() { - cat <<'USAGE' -Run Terminal-Bench sample with Maka heavy-task mode. - -Usage: - terminal-bench-smoke/run-terminal-bench-sample-heavy.sh [options] - -This is a thin wrapper over run-terminal-bench-sample.sh. By default it runs: - run-terminal-bench-sample.sh --profile maka-heavy ... - -Options: - --compare-opencode Run maka-heavy and opencode sequentially for comparison - --task PATTERN Harbor task pattern - --n-tasks N Pick N tasks instead of using --task - --job-name NAME Harbor job name - --model MODEL Override MAKA_MODEL - --steps N Override MAKA_MAX_STEPS - --agent-timeout-sec N Override MAKA_HARBOR_AGENT_TIMEOUT_SEC - --dataset NAME Override dataset name - --dataset-version VERSION Override dataset version - --dry-run Generate config and print command without running Harbor - -h, --help Show this help - -Examples: - terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --n-tasks 10 - terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --task '*qemu-startup' - terminal-bench-smoke/run-terminal-bench-sample-heavy.sh --compare-opencode --n-tasks 10 -USAGE -} - -if [ ! -x "$RUNNER" ]; then - echo "Runner not found or not executable: $RUNNER" >&2 - exit 1 -fi - -while [ "$#" -gt 0 ]; do - case "$1" in - --compare-opencode) - COMPARE_OPENCODE=1 - shift - ;; - --profile|--compare|--compare-profiles) - echo "run-terminal-bench-sample-heavy.sh fixes the profile to maka-heavy; use the base runner for $1" >&2 - exit 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - ARGS+=("$1") - shift - ;; - esac -done - -if [ "$COMPARE_OPENCODE" -eq 1 ]; then - exec "$RUNNER" --compare --compare-profiles maka-heavy,opencode "${ARGS[@]}" -fi - -exec "$RUNNER" --profile maka-heavy "${ARGS[@]}" diff --git a/terminal-bench-smoke/run-terminal-bench-sample.sh b/terminal-bench-smoke/run-terminal-bench-sample.sh deleted file mode 100755 index b08f48a68a..0000000000 --- a/terminal-bench-smoke/run-terminal-bench-sample.sh +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -MANIFEST_PATH="${SCRIPT_DIR}/terminal-bench-sample-runs.json" -HARBOR_BIN="${HARBOR_BIN:-${SCRIPT_DIR}/harbor-venv/bin/harbor}" - -PROFILE="maka-basic" -COMPARE=0 -COMPARE_PROFILES="maka-basic,opencode" -TASK_PATTERN="" -JOB_NAME="" -MODEL="" -MAX_STEPS="" -AGENT_TIMEOUT_SEC="" -N_TASKS="" -DATASET_NAME="" -DATASET_VERSION="" -DRY_RUN=0 - -usage() { - cat <<'USAGE' -Run a structured Terminal-Bench sample job through the local Harbor smoke harness. - -Usage: - terminal-bench-smoke/run-terminal-bench-sample.sh [options] - -Profiles: - maka-basic Maka Harbor bridge, non-autonomous, DeepSeek V4 Pro (default) - maka-heavy Maka task-run heavy-task bridge for trace/evidence experiments - maka-heavy-prune - Maka heavy-task bridge with autonomous prior-attempt runtime replay - and stale tool-result archive pruning enabled - maka-prune-default - Post-#621 default prune pipeline with continuation (stale A/B B arm, - retrieval A/B A arm) - maka-stale-off - maka-prune-default with stale prune explicitly off (stale A/B A arm) - maka-retrieval-on - maka-prune-default plus eager archive retrieval (retrieval A/B B arm) - opencode OpenCode Harbor wrapper - oracle Harbor oracle agent for cheap wrapper/dataset smoke tests - -Options: - --profile NAME Run profile: maka-basic, maka-heavy, maka-heavy-prune, maka-prune-default, maka-stale-off, maka-retrieval-on, opencode, oracle - --compare Run comparison profiles sequentially (default: maka-basic,opencode) - --compare-profiles LIST Comma-separated profiles for --compare - --task PATTERN Harbor task pattern (default: *sqlite-with-gcov) - --n-tasks N Pick N tasks instead of using --task - --job-name NAME Harbor job name (default: generated with timestamp) - --model MODEL Override model. For Maka this sets MAKA_MODEL; for OpenCode it sets model_name. - --steps N Override MAKA_MAX_STEPS for Maka profiles - --agent-timeout-sec N Override MAKA_HARBOR_AGENT_TIMEOUT_SEC for Maka profiles - --dataset NAME Override dataset name (default: terminal-bench-sample) - --dataset-version VERSION Override dataset version (default: 2.0) - --dry-run Generate and print config path/command without running Harbor - -h, --help Show this help - -Examples: - terminal-bench-smoke/run-terminal-bench-sample.sh --profile oracle --n-tasks 1 - terminal-bench-smoke/run-terminal-bench-sample.sh --profile maka-basic --task '*sqlite-with-gcov' - terminal-bench-smoke/run-terminal-bench-sample.sh --compare --task '*sqlite-with-gcov' - terminal-bench-smoke/run-terminal-bench-sample.sh --profile maka-heavy --job-name maka-sample-heavy-$(date +%Y%m%d%H%M%S) -USAGE -} - -while [ "$#" -gt 0 ]; do - case "$1" in - --profile) - PROFILE="${2:?missing value for --profile}" - shift 2 - ;; - --compare) - COMPARE=1 - shift - ;; - --compare-profiles) - COMPARE=1 - COMPARE_PROFILES="${2:?missing value for --compare-profiles}" - shift 2 - ;; - --task) - TASK_PATTERN="${2:?missing value for --task}" - shift 2 - ;; - --n-tasks) - N_TASKS="${2:?missing value for --n-tasks}" - shift 2 - ;; - --job-name) - JOB_NAME="${2:?missing value for --job-name}" - shift 2 - ;; - --model) - MODEL="${2:?missing value for --model}" - shift 2 - ;; - --steps) - MAX_STEPS="${2:?missing value for --steps}" - shift 2 - ;; - --agent-timeout-sec) - AGENT_TIMEOUT_SEC="${2:?missing value for --agent-timeout-sec}" - shift 2 - ;; - --dataset) - DATASET_NAME="${2:?missing value for --dataset}" - shift 2 - ;; - --dataset-version) - DATASET_VERSION="${2:?missing value for --dataset-version}" - shift 2 - ;; - --dry-run) - DRY_RUN=1 - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "unknown option: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -if [ ! -x "$HARBOR_BIN" ]; then - echo "Harbor binary not found or not executable: $HARBOR_BIN" >&2 - exit 1 -fi - -mkdir -p "${SCRIPT_DIR}/generated-configs" - -generate_config() { - local profile_name="$1" - local job_name="$2" - - MANIFEST_PATH="$MANIFEST_PATH" \ - PROFILE="$profile_name" \ - TASK_PATTERN="$TASK_PATTERN" \ - JOB_NAME="$job_name" \ - MODEL="$MODEL" \ - MAX_STEPS="$MAX_STEPS" \ - AGENT_TIMEOUT_SEC="$AGENT_TIMEOUT_SEC" \ - N_TASKS="$N_TASKS" \ - DATASET_NAME="$DATASET_NAME" \ - DATASET_VERSION="$DATASET_VERSION" \ - node <<'NODE' -const fs = require('fs'); -const path = require('path'); - -const manifestPath = process.env.MANIFEST_PATH; -const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); -const defaults = manifest.defaults || {}; -const profileName = process.env.PROFILE || 'maka-basic'; -const profile = manifest.profiles && manifest.profiles[profileName]; -if (!profile) { - const names = Object.keys(manifest.profiles || {}).join(', '); - throw new Error(`unknown profile "${profileName}". Available profiles: ${names}`); -} - -function env(name) { - const value = process.env[name]; - return value && value.length ? value : null; -} - -function slug(value) { - return String(value) - .replace(/^\*/, '') - .replace(/[^A-Za-z0-9_.-]+/g, '-') - .replace(/^-+|-+$/g, '') - .slice(0, 80) || 'sample'; -} - -function retryConfig() { - return { - max_retries: Number(defaults.retryMaxRetries || 0), - include_exceptions: null, - exclude_exceptions: [ - 'AgentTimeoutError', - 'VerifierOutputParseError', - 'VerifierTimeoutError', - 'RewardFileNotFoundError', - 'RewardFileEmptyError', - ], - wait_multiplier: 1.0, - min_wait_sec: 1.0, - max_wait_sec: 60.0, - }; -} - -const taskPattern = env('TASK_PATTERN') || defaults.taskPattern || '*sqlite-with-gcov'; -const datasetName = env('DATASET_NAME') || (defaults.dataset && defaults.dataset.name) || 'terminal-bench-sample'; -const datasetVersion = env('DATASET_VERSION') || (defaults.dataset && defaults.dataset.version) || '2.0'; -const nTasksRaw = env('N_TASKS'); -const nTasks = nTasksRaw ? Number(nTasksRaw) : null; -if (nTasksRaw && (!Number.isInteger(nTasks) || nTasks <= 0)) { - throw new Error(`--n-tasks must be a positive integer, got ${nTasksRaw}`); -} - -const explicitJobName = env('JOB_NAME'); -const jobName = explicitJobName || [ - profileName, - 'terminal-bench-sample', - nTasks ? `n${nTasks}` : slug(taskPattern), - new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, 'Z'), -].join('-'); - -const agent = profile.agent || {}; -const agentEnv = { ...(agent.env || {}) }; -let agentModelName = agent.modelName || null; -const modelOverride = env('MODEL'); -if (modelOverride) { - if (profileName.startsWith('maka-')) { - agentEnv.MAKA_MODEL = modelOverride; - } else { - agentModelName = modelOverride; - } -} - -const maxSteps = env('MAX_STEPS'); -if (maxSteps) agentEnv.MAKA_MAX_STEPS = maxSteps; -const agentTimeoutSec = env('AGENT_TIMEOUT_SEC'); -if (agentTimeoutSec) agentEnv.MAKA_HARBOR_AGENT_TIMEOUT_SEC = agentTimeoutSec; -if (profileName.startsWith('maka-') && !agentEnv.MAKA_BENCHMARK_DATASET) { - agentEnv.MAKA_BENCHMARK_DATASET = env('MAKA_BENCHMARK_DATASET') || datasetName; -} - -const extraInstructionPaths = Object.prototype.hasOwnProperty.call(profile, 'extraInstructionPaths') - ? profile.extraInstructionPaths - : (defaults.modelExtraInstructionPaths || []); - -const config = { - job_name: jobName, - jobs_dir: defaults.jobsDir || 'terminal-bench-smoke/jobs', - n_attempts: Number(defaults.nAttempts || 1), - timeout_multiplier: Number(defaults.timeoutMultiplier || 1.0), - agent_timeout_multiplier: profile.agentTimeoutMultiplier === undefined ? null : profile.agentTimeoutMultiplier, - verifier_timeout_multiplier: null, - agent_setup_timeout_multiplier: null, - environment_build_timeout_multiplier: null, - debug: false, - n_concurrent_trials: Number(defaults.nConcurrentTrials || 1), - quiet: false, - retry: retryConfig(), - environment: { - type: 'docker', - import_path: null, - force_build: false, - delete: true, - cpu_enforcement_policy: 'auto', - memory_enforcement_policy: 'auto', - override_cpus: null, - override_memory_mb: null, - override_storage_mb: null, - override_gpus: null, - override_tpu: null, - mounts: null, - extra_docker_compose: [], - env: {}, - kwargs: {}, - extra_allowed_hosts: [], - }, - verifier: { - override_timeout_sec: null, - max_timeout_sec: null, - env: {}, - disable: false, - }, - metrics: [], - agents: [ - { - name: agent.name || null, - import_path: agent.importPath || null, - model_name: agentModelName, - skills: [], - override_timeout_sec: null, - override_setup_timeout_sec: null, - max_timeout_sec: null, - extra_allowed_hosts: [], - kwargs: agent.kwargs || {}, - env: agentEnv, - mcp_servers: [], - }, - ], - datasets: [ - { - path: null, - name: datasetName, - version: datasetVersion, - ref: null, - registry_url: null, - registry_path: null, - overwrite: false, - download_dir: null, - task_names: nTasks ? null : [taskPattern], - exclude_task_names: null, - n_tasks: nTasks, - }, - ], - tasks: [], - artifacts: [], - extra_instruction_paths: extraInstructionPaths, - plugins: [], -}; - -const generatedDir = path.resolve(path.dirname(manifestPath), 'generated-configs'); -fs.mkdirSync(generatedDir, { recursive: true }); -const configPath = path.join(generatedDir, `${jobName}.json`); -fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); -process.stdout.write(configPath); -NODE -} - -run_one() { - local profile_name="$1" - local job_name="$2" - local config_path - config_path="$(generate_config "$profile_name" "$job_name")" - - echo "Generated Harbor config: $config_path" - echo "Profile: $profile_name" - echo "Run command:" - echo " PYTHONPATH=terminal-bench-smoke $HARBOR_BIN run --config $config_path --yes" - - if [ "$DRY_RUN" -eq 1 ]; then - return 0 - fi - - PYTHONPATH="${SCRIPT_DIR}${PYTHONPATH:+:${PYTHONPATH}}" "$HARBOR_BIN" run --config "$config_path" --yes -} - -cd "$WORKSPACE_DIR" - -if [ "$COMPARE" -eq 1 ]; then - IFS=',' read -r -a profiles <<< "$COMPARE_PROFILES" - for profile_name in "${profiles[@]}"; do - profile_name="${profile_name#"${profile_name%%[![:space:]]*}"}" - profile_name="${profile_name%"${profile_name##*[![:space:]]}"}" - if [ -z "$profile_name" ]; then - continue - fi - compare_job_name="" - if [ -n "$JOB_NAME" ]; then - compare_job_name="${JOB_NAME}-${profile_name}" - fi - run_one "$profile_name" "$compare_job_name" - done -else - run_one "$PROFILE" "$JOB_NAME" -fi diff --git a/terminal-bench-smoke/terminal-bench-sample-runs.json b/terminal-bench-smoke/terminal-bench-sample-runs.json deleted file mode 100644 index afe8da1463..0000000000 --- a/terminal-bench-smoke/terminal-bench-sample-runs.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "schemaVersion": 1, - "description": "Structured Terminal-Bench sample run profiles for the local Harbor smoke harness.", - "defaults": { - "jobsDir": "terminal-bench-smoke/jobs", - "generatedConfigDir": "terminal-bench-smoke/generated-configs", - "dataset": { - "name": "terminal-bench-sample", - "version": "2.0" - }, - "taskPattern": "*sqlite-with-gcov", - "nAttempts": 1, - "nConcurrentTrials": 1, - "timeoutMultiplier": 1.0, - "retryMaxRetries": 0, - "modelExtraInstructionPaths": [] - }, - "profiles": { - "maka-basic": { - "description": "Maka Harbor bridge against terminal-bench-sample, matching the successful sqlite-with-gcov DeepSeek V4 Pro sample run shape.", - "agentTimeoutMultiplier": 4.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "0", - "MAKA_HARBOR_AUTONOMOUS": "0", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "80", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" - } - } - }, - "maka-heavy": { - "description": "Maka task-run heavy-task bridge for public sample trace/evidence experiments.", - "agentTimeoutMultiplier": 8.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "1", - "MAKA_HARBOR_AUTONOMOUS": "0", - "MAKA_HEAVY_TASK_MODE": "1", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "100", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "7200" - } - } - }, - "maka-heavy-prune": { - "description": "Maka heavy-task bridge with autonomous prior-attempt runtime replay and stale tool-result archive pruning enabled.", - "agentTimeoutMultiplier": 8.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "1", - "MAKA_HARBOR_AUTONOMOUS": "1", - "MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT": "1", - "MAKA_HEAVY_TASK_MODE": "1", - "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE": "on", - "MAKA_CONTEXT_STALE_TOOL_RESULT_MAX_ESTIMATED_TOKENS": "2048", - "MAKA_CONTEXT_STALE_TOOL_RESULT_MIN_RECENT_TURNS_FULL": "0", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_PRUNE": "on", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MAX_ESTIMATED_TOKENS": "2048", - "MAKA_CONTEXT_ACTIVE_TOOL_RESULT_MIN_STEP_NUMBER": "1", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT": "on", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_STEP_NUMBER": "2", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_HIGH_WATER_RATIO": "0.5", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_ACTIVE_ESTIMATED_TOKENS": "16384", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MIN_RECENT_MESSAGES": "4", - "MAKA_CONTEXT_ACTIVE_FULL_COMPACT_MAX_SUMMARY_ESTIMATED_TOKENS": "1024", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "100", - "MAKA_AUTONOMOUS_MAX_ATTEMPTS": "3", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "7200" - } - } - }, - "maka-prune-default": { - "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B.", - "agentTimeoutMultiplier": 4.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "0", - "MAKA_HARBOR_AUTONOMOUS": "0", - "MAKA_HARBOR_CONTINUATION": "on", - "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", - "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "50", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" - } - } - }, - "maka-stale-off": { - "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B.", - "agentTimeoutMultiplier": 4.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "0", - "MAKA_HARBOR_AUTONOMOUS": "0", - "MAKA_HARBOR_CONTINUATION": "on", - "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", - "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", - "MAKA_CONTEXT_STALE_TOOL_RESULT_PRUNE": "off", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "50", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" - } - } - }, - "maka-retrieval-on": { - "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B.", - "agentTimeoutMultiplier": 4.0, - "agent": { - "importPath": "maka_harbor_agent:MakaHarborAgent", - "env": { - "MAKA_HARBOR_USE_TASK_RUN": "0", - "MAKA_HARBOR_AUTONOMOUS": "0", - "MAKA_HARBOR_CONTINUATION": "on", - "MAKA_HARBOR_CONTINUATION_MAX_TURNS": "3", - "MAKA_HARBOR_CONTINUATION_MAX_TOTAL_RUNTIME_STEPS": "150", - "MAKA_CONTEXT_ARCHIVE_RETRIEVAL": "on", - "MAKA_MODEL": "deepseek-v4-pro", - "MAKA_MAX_STEPS": "50", - "MAKA_HARBOR_AGENT_TIMEOUT_SEC": "3600" - } - } - }, - "opencode": { - "description": "OpenCode Harbor wrapper against terminal-bench-sample for Maka/OpenCode comparison.", - "agentTimeoutMultiplier": 4.0, - "agent": { - "importPath": "opencode_title_harbor_agent:OpenCodeTitleAgent", - "modelName": "deepseek/deepseek-v4-pro", - "env": {} - } - }, - "oracle": { - "description": "Harbor built-in oracle agent for cheap wrapper/dataset smoke tests.", - "agentTimeoutMultiplier": null, - "extraInstructionPaths": [], - "agent": { - "name": "oracle", - "env": {} - } - } - } -} From d404eb8cf1d624185346815c3344688ad983b83b Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 17:58:27 +0800 Subject: [PATCH 4/8] docs(headless): drop provenance comments from the merged adapter --- packages/headless/harbor/maka_agent.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index 4b41cf73d9..5757505493 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -122,9 +122,8 @@ def name() -> str: def _harbor_mode(self) -> str: """cell (default) runs a RuntimeRunner cell; task-run runs the full task-run controller on the host and bridges tool execution into the - container via the shared _ToolExecutorServer. The mode switch keeps the - heavy-task / autonomous experiment path that used to live in the - terminal-bench-smoke fork on the single authoritative adapter.""" + container via the shared _ToolExecutorServer. task-run is the + heavy-task / autonomous experiment path.""" mode = (self._get_env("MAKA_HARBOR_MODE") or "cell").strip() if mode not in ("cell", "task-run"): raise RuntimeError(f"MAKA_HARBOR_MODE must be cell or task-run, got {mode!r}") @@ -516,8 +515,7 @@ async def _run_task_run_host( context: AgentContext, ) -> None: """Run the full task-run controller on the host, bridging tool execution - into the task container. Ported from the terminal-bench-smoke fork so the - heavy-task / autonomous experiment path is preserved end to end.""" + into the task container.""" self.logs_dir.mkdir(parents=True, exist_ok=True) env = os.environ.copy() @@ -1220,7 +1218,7 @@ def _apply_trial_pricing(agent: MakaAgent, token_summary: dict[str, Any]) -> Non # --------------------------------------------------------------------------- -# task-run host mode helpers (ported from the terminal-bench-smoke fork) +# task-run host mode helpers # --------------------------------------------------------------------------- From 7bbbe1a4f6ed19c617fc1f34f75cb5a2b3f65f10 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 18:42:08 +0800 Subject: [PATCH 5/8] fix(headless): correct Harbor bridge exec/workdir/reclaim contract and drop make-mips shortcut Route bridged tool commands and the workdir probe through a bare environment.exec instead of exec_as_agent: a non-zero exit is now a successful HTTP 200 transport response carrying the real exit code (exitCode + returnCode), the agent's _extra_env no longer leaks into the model's command environment, and the per-command timeout defaults to 120s (matching the in-container executor floor) instead of borrowing the whole cell budget. Read the exit status from Harbor 0.13.2's return_code field. The workdir probe now treats a non-zero pwd as "candidate unavailable" and continues, so /app, /workspace and / are reachable again. A non-zero runner subprocess is flagged inside the executor scope via mark_infra_failure() so teardown reclaims orphaned scoped background processes rather than preserving them as a completed run; a clean exit still preserves verifier-visible services. Remove the unused MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE direct-smoke branch (no callers repo-wide). Cover the bridge/workdir/reclaim contract with a behavior-level test that exercises the real Harbor _ToolExecutorServer against a fake environment, skipping where Harbor is unavailable. --- packages/headless/harbor/maka_agent.py | 195 ++++----------- .../src/__tests__/harbor-adapter.test.ts | 234 ++++++++++++++++-- 2 files changed, 256 insertions(+), 173 deletions(-) diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index 5757505493..8949cbddde 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -28,6 +28,13 @@ _COMMAND_SCOPE_ENV = "MAKA_HARBOR_COMMAND_SCOPE" _COMMAND_SCOPE_ROOT = "/tmp/maka-harbor-command-scopes" +# Default wall-clock budget for a single bridged tool command when the client +# does not request its own timeout. Matches the in-container executor floor +# (HARBOR_CELL_DEFAULT_COMMAND_TIMEOUT_MS = 120_000) rather than the whole-cell +# budget, so an individual command cannot silently borrow the 15-minute cell +# deadline. +_BRIDGE_DEFAULT_TIMEOUT_SEC = 120 + # Location of this adapter and the repo it ships in. The task-run host mode # spawns `node /packages/headless/dist/cli.js` on the host, so it needs to # find the built headless CLI relative to this file when no explicit repo root @@ -560,21 +567,6 @@ async def _run_task_run_host( }, ) - if env.get("MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE") == "1": - await self._run_direct_make_mips_smoke( - environment, - context=context, - env=env, - task_workdir=task_workdir, - workdir_probe=workdir_probe, - stdout_path=stdout_path, - stderr_path=stderr_path, - status_path=status_path, - task_run_out_dir=task_run_out_dir, - started_at=started_at, - ) - return - timeout_sec = int(env.get("MAKA_HARBOR_AGENT_TIMEOUT_SEC", "1800")) proc: asyncio.subprocess.Process | None = None async with _ToolExecutorServer(self, environment) as executor: @@ -662,6 +654,13 @@ async def _run_task_run_host( ) raise + # A non-zero runner exit is an infrastructure failure. Flag it inside + # the executor scope so __aexit__ reclaims scoped background processes + # instead of preserving them as if the run had completed. A clean exit + # (return code 0) still preserves verifier-visible services. + if proc is not None and proc.returncode != 0: + executor.mark_infra_failure() + parsed = self._parse_node_result(stdout) assert proc is not None self._write_status( @@ -720,75 +719,6 @@ async def _run_task_run_host( if proc.returncode != 0: raise RuntimeError(f"Maka Harbor task-run failed; see {stderr_path}") - async def _run_direct_make_mips_smoke( - self, - environment: BaseEnvironment, - *, - context: AgentContext, - env: dict[str, str], - task_workdir: str, - workdir_probe: list[dict[str, Any]], - stdout_path: Path, - stderr_path: Path, - status_path: Path, - task_run_out_dir: Path, - started_at: str, - ) -> None: - result = await self.exec_as_agent( - environment, - command=_direct_make_mips_smoke_command(), - cwd=task_workdir, - timeout_sec=30, - ) - return_code = _exec_exit_code(result) - direct_payload = { - "ok": return_code == 0, - "status": "completed" if return_code == 0 else "failed", - "mode": "direct-make-mips-smoke", - "cwd": task_workdir, - "returnCode": return_code, - "stdout": _exec_stdout(result), - "stderr": _exec_stderr(result), - } - stdout_path.write_text(json.dumps(direct_payload) + "\n", encoding="utf-8") - stderr_path.write_text("", encoding="utf-8") - self._write_status( - status_path, - { - "status": direct_payload["status"], - "startedAt": started_at, - "finishedAt": _utc_now(), - "mode": direct_payload["mode"], - "returnCode": return_code, - "stdoutLog": str(stdout_path), - "stderrLog": str(stderr_path), - "taskRunOutDir": str(task_run_out_dir), - "resolvedCwd": task_workdir, - "workdirProbe": workdir_probe, - "runnerEnv": _runner_env_summary(env), - }, - ) - context.metadata = { - **(context.metadata or {}), - "maka_harbor": { - "return_code": return_code, - "stdout_log": str(stdout_path), - "stderr_log": str(stderr_path), - "status": direct_payload["status"], - "model": "direct-make-mips-smoke", - "max_steps": 0, - "event_count": 0, - "message_count": 0, - "llm_call_count": 0, - "tool_call_count": 1, - "error": None if return_code == 0 else _exec_stderr(result), - "resolved_cwd": task_workdir, - "workdir_probe": workdir_probe, - }, - } - if return_code != 0: - raise RuntimeError(f"direct make-mips smoke setup failed; see {stdout_path}") - async def _resolve_task_workdir( self, environment: BaseEnvironment, @@ -806,8 +736,10 @@ async def _resolve_task_workdir( if marker in seen: continue seen.add(marker) - result = await self.exec_as_agent( - environment, + # Probe with a bare exec: an invalid candidate directory makes `pwd` + # exit non-zero, and that must skip to the next candidate rather than + # abort the whole probe (exec_as_agent would raise on the first miss). + result = await environment.exec( command="pwd", cwd=candidate, timeout_sec=10, @@ -938,6 +870,7 @@ def __init__(self, agent: MakaAgent, environment: BaseEnvironment) -> None: self._futures: set[concurrent.futures.Future[Any]] = set() self._futures_lock = threading.Lock() self._accepting_requests = False + self._infra_failure = False self.token = secrets.token_urlsafe(32) self.command_scope = secrets.token_urlsafe(24) self.url = "" @@ -962,16 +895,25 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib c self._thread.start() return self + def mark_infra_failure(self) -> None: + """Record that the run failed for infrastructure reasons so teardown + reclaims scoped background processes even when the `async with` block + exits without raising. A non-zero runner subprocess is such a failure: + leaving its orphaned background services alive would masquerade as a + successful completion to the verifier.""" + self._infra_failure = True + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: stop_error: BaseException | None = None cleanup_error: BaseException | None = None + reclaim = exc_type is not None or self._infra_failure try: cleanup_error = await self._stop_server( - reclaim_scoped_processes=exc_type is not None + reclaim_scoped_processes=reclaim ) except BaseException as error: stop_error = error - if stop_error is not None and exc_type is None: + if stop_error is not None and not reclaim: cleanup_error = await self._cleanup_all_scoped_processes() if stop_error is not None: raise stop_error @@ -1054,14 +996,17 @@ def _handle_post(self, handler: BaseHTTPRequestHandler) -> None: raise ValueError("command is required") cwd = payload.get("cwd") timeout_ms = payload.get("timeoutMs") - timeout_sec = _timeout_sec(timeout_ms) + timeout_sec = _timeout_sec(timeout_ms) or _BRIDGE_DEFAULT_TIMEOUT_SEC assert self._loop is not None with self._futures_lock: if not self._accepting_requests: raise RuntimeError("tool executor is shutting down") + # Run the bridged tool command as a bare container exec, not via + # exec_as_agent: a non-zero exit is a *successful* transport + # response (the tool ran and reported a failure), and the agent's + # _extra_env must never leak into the model's command environment. future = asyncio.run_coroutine_threadsafe( - self._agent.exec_as_agent( - self._environment, + self._environment.exec( command=_scoped_command( command, self.command_scope, @@ -1074,9 +1019,11 @@ def _handle_post(self, handler: BaseHTTPRequestHandler) -> None: ) self._futures.add(future) future.add_done_callback(self._discard_future) - result = future.result(timeout=(timeout_sec or self._agent._cell_timeout_sec()) + 30) + result = future.result(timeout=timeout_sec + 30) + return_code = _exec_exit_code(result) _write_http(handler, 200, { - "exitCode": _exec_exit_code(result), + "exitCode": return_code, + "returnCode": return_code, "stdout": _exec_stdout(result), "stderr": _exec_stderr(result), }) @@ -1150,7 +1097,9 @@ def _exec_stderr(result: Any) -> str: def _exec_exit_code(result: Any) -> int: - for name in ("exit_code", "exitCode", "returncode"): + # Harbor 0.13.2 ExecResult exposes the exit status as `return_code`; the + # other names stay for compatibility with older/stubbed exec results. + for name in ("return_code", "exit_code", "exitCode", "returncode"): value = getattr(result, name, None) if isinstance(value, int): return value @@ -1394,7 +1343,6 @@ def _runner_env_summary(env: dict[str, str]) -> dict[str, str]: "MAKA_AUTONOMOUS_MAX_ATTEMPTS", "MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS", "MAKA_AUTONOMOUS_MAX_WALL_TIME_MS", - "MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE", ] return {key: env[key] for key in allowed_keys if key in env} @@ -1416,60 +1364,3 @@ def _int_or_none(value: Any) -> int | None: return int(value) except (TypeError, ValueError): return None - - -def _direct_make_mips_smoke_command() -> str: - return r"""cat > /app/vm.js <<'VMEOF' -#!/usr/bin/env node -const fs = require('fs'); - -function writeFallbackBmp(outPath) { - const width = 320; - const height = 200; - const rowSize = Math.ceil((width * 3) / 4) * 4; - const pixelSize = rowSize * height; - const fileSize = 54 + pixelSize; - const buf = Buffer.alloc(fileSize, 0); - buf.write('BM', 0, 'ascii'); - buf.writeUInt32LE(fileSize, 2); - buf.writeUInt32LE(54, 10); - buf.writeUInt32LE(40, 14); - buf.writeInt32LE(width, 18); - buf.writeInt32LE(height, 22); - buf.writeUInt16LE(1, 26); - buf.writeUInt16LE(24, 28); - buf.writeUInt32LE(pixelSize, 34); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const i = 54 + y * rowSize + x * 3; - buf[i] = (x + y) & 255; - buf[i + 1] = (2 * x) & 255; - buf[i + 2] = (2 * y) & 255; - } - } - fs.writeFileSync(outPath, buf); -} - -function writeFrame() { - if (fs.existsSync('/tests/reference.jpg')) { - fs.copyFileSync('/tests/reference.jpg', '/tmp/frame.bmp'); - return; - } - writeFallbackBmp('/tmp/frame.bmp'); -} - -console.log('I_InitGraphics: DOOM screen size: w x h: 320 x 200'); -writeFrame(); -setInterval(() => {}, 1000); -VMEOF -chmod +x /app/vm.js -node /app/vm.js >/tmp/direct-make-mips-smoke.out 2>&1 & -pid=$! -for i in $(seq 1 30); do - test -s /tmp/frame.bmp && break - sleep 1 -done -kill "$pid" 2>/dev/null || true -wait "$pid" 2>/dev/null || true -test -s /tmp/frame.bmp -""" diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index 56711b3c33..474014a46e 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -1,9 +1,9 @@ import assert from 'node:assert/strict'; import { execFile, spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, realpathSync, rmSync } from 'node:fs'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { resolve } from 'node:path'; +import { dirname, resolve } from 'node:path'; import { describe, test, type TestContext } from 'node:test'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; @@ -306,14 +306,36 @@ describe('Harbor adapter contract', () => { assert.match(source, /MAKA_HARBOR_RUNNER_ENV_FILE/); assert.match(source, /maka-harbor\.status\.json/); assert.match(source, /maka-harbor\.stdout\.json/); - assert.match(source, /MAKA_HARBOR_DIRECT_MAKE_MIPS_SMOKE/); + // The make-mips direct-smoke shortcut was removed (no callers repo-wide). + assert.doesNotMatch(source, /MAKE_MIPS|_run_direct_make_mips_smoke|_direct_make_mips_smoke_command/); // Reuses the shared tool executor bridge rather than a forked HTTP server. assert.match(source, /async with _ToolExecutorServer\(self, environment\) as executor:/); + // A non-zero runner subprocess is flagged as an infra failure inside the + // executor scope so teardown reclaims orphaned scoped background processes. + assert.match(source, /executor\.mark_infra_failure\(\)/); + // The bridged tool exec is a bare container exec (real exit code returned as + // a 200), not exec_as_agent (which raises on non-zero and injects agent env). + assert.match(source, /self\._environment\.exec\(\s*\n\s*command=_scoped_command\(/); + assert.match(source, /"returnCode": return_code,/); // Benchmark-vs-infra exit policy comes from the node CLI's parsed result. assert.match(source, /benchmarkFailureShouldThrow/); assert.doesNotMatch(source, /class MakaHarborAgent/); }); + test('maka_agent.py bridge exec contract holds against the real Harbor _ToolExecutorServer', (t: TestContext) => { + const python = harborPython(); + if (!python) { + t.skip('Harbor 0.13.2 python is not available (CI has no harbor)'); + return; + } + const result = spawnSync(python, ['-c', pythonBridgeContractScript(repoRoot)], { + cwd: repoRoot, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /bridge-contract ok/); + }); + test('run-prompt-optimization.mjs wires the headless run API with a key file, not a raw key', async () => { const source = await readRepoFile('packages/headless/harbor/run-prompt-optimization.mjs'); assert.match(source, /runPromptOptimizationRun/); @@ -743,6 +765,170 @@ function sha256(char: string): string { return `sha256:${char.repeat(64)}`; } +/** Locate a Python interpreter with Harbor importable, or null when none + * exists (e.g. CI). Harbor is installed as a uv tool, so its interpreter sits + * next to the `harbor` launcher on PATH. */ +function harborPython(): string | null { + const candidates: string[] = []; + if (process.env.MAKA_HARBOR_PYTHON) candidates.push(process.env.MAKA_HARBOR_PYTHON); + const located = spawnSync('bash', ['-lc', 'command -v harbor'], { encoding: 'utf8' }); + if (located.status === 0 && located.stdout.trim()) { + try { + const binDir = dirname(realpathSync(located.stdout.trim())); + candidates.push(resolve(binDir, 'python'), resolve(binDir, 'python3')); + } catch { + // Unresolvable launcher — fall through to remaining candidates. + } + } + for (const python of candidates) { + const check = spawnSync(python, ['-c', 'import harbor'], { encoding: 'utf8' }); + if (check.status === 0) return python; + } + return null; +} + +function pythonBridgeContractScript(root: string): string { + return String.raw` +import asyncio +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import types +import urllib.request +from pathlib import Path + +root = Path(${JSON.stringify(root)}) +sys.path.insert(0, str(root / "packages" / "headless" / "harbor")) + +import maka_agent as m +from maka_agent import MakaAgent +from harbor.environments.base import ExecResult + + +def post(url, token, body): + request = urllib.request.Request( + url + "/exec", + data=json.dumps(body).encode("utf-8"), + headers={"authorization": "Bearer " + token, "content-type": "application/json"}, + method="POST", + ) + resp = urllib.request.urlopen(request, timeout=5) + return resp.status, json.loads(resp.read()) + + +class FakeExecEnv: + def __init__(self): + self.calls = [] + + async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None): + self.calls.append({"command": command, "cwd": cwd, "env": env, "timeout_sec": timeout_sec}) + return ExecResult(stdout="OUT", stderr="ERR", return_code=17) + + +async def fix1_bridge_exec_contract(): + with tempfile.TemporaryDirectory() as tmp: + agent = MakaAgent(Path(tmp), extra_env={"MAKA_LEAK": "should-not-leak"}) + env = FakeExecEnv() + async with m._ToolExecutorServer(agent, env) as server: + # A non-zero exit is a successful transport response: HTTP 200 with + # the real exit code echoed on both exitCode and returnCode. + status, body = await asyncio.to_thread(post, server.url, server.token, {"command": "false"}) + assert status == 200, status + assert body["exitCode"] == 17, body + assert body["returnCode"] == 17, body + assert body["stdout"] == "OUT" and body["stderr"] == "ERR", body + # No explicit timeout -> 120s default, and no agent env is injected + # (exec_as_agent would have merged _extra_env into the command env). + call = env.calls[-1] + assert call["timeout_sec"] == 120, call + assert call["env"] is None, call + assert "should-not-leak" not in json.dumps(call), call + assert "MAKA_HARBOR_COMMAND_SCOPE=" in call["command"], call + # Explicit timeoutMs takes precedence, rounded up to whole seconds. + await asyncio.to_thread(post, server.url, server.token, {"command": "x", "timeoutMs": 4500}) + assert env.calls[-1]["timeout_sec"] == 5, env.calls[-1] + + +class WorkdirProbeEnv: + task_env_config = types.SimpleNamespace(workdir=None) + + def __init__(self): + self.attempts = [] + + async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None): + self.attempts.append(cwd) + if cwd == "/app": + return ExecResult(stdout="/app\n", stderr="", return_code=0) + return ExecResult(stdout="", stderr="no such directory", return_code=1) + + +async def fix2_workdir_probe_falls_back(): + with tempfile.TemporaryDirectory() as tmp: + agent = MakaAgent(Path(tmp)) + env = WorkdirProbeEnv() + resolved, probes = await agent._resolve_task_workdir(env) + assert resolved == "/app", (resolved, env.attempts) + # The first candidate (default cwd) exited non-zero but the probe + # continued instead of aborting. + assert env.attempts[0] is None and probes[0]["return_code"] == 1, (env.attempts, probes) + assert "/app" in env.attempts, env.attempts + + +class LocalShellEnv: + async def exec(self, command, cwd=None, env=None, timeout_sec=None, user=None): + process = await asyncio.create_subprocess_exec("bash", "-lc", command) + await process.wait() + return ExecResult(stdout="", stderr="", return_code=process.returncode) + + +async def _run_with_scoped_service(mark_infra_failure): + with tempfile.TemporaryDirectory() as tmp: + agent = MakaAgent(Path(tmp)) + server = m._ToolExecutorServer(agent, LocalShellEnv()) + scope_dir = Path(m._COMMAND_SCOPE_ROOT) / server.command_scope + scope_dir.mkdir(parents=True, exist_ok=True) + process = subprocess.Popen( + ["sleep", "30"], + start_new_session=True, + env={"PATH": os.environ.get("PATH", "")}, + ) + (scope_dir / "service.pgid").write_text(str(process.pid) + "\n", encoding="utf-8") + try: + async with server: + if mark_infra_failure: + server.mark_infra_failure() + deadline = time.time() + 2 + while process.poll() is None and time.time() < deadline: + time.sleep(0.02) + return process.poll() + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + if scope_dir.exists(): + for path in scope_dir.glob("*"): + path.unlink() + scope_dir.rmdir() + + +async def fix3_infra_failure_reclaims_scoped_processes(): + reclaimed = await _run_with_scoped_service(mark_infra_failure=True) + assert reclaimed is not None, "infra failure did not reclaim the scoped background process" + preserved = await _run_with_scoped_service(mark_infra_failure=False) + assert preserved is None, "clean exit killed a verifier-visible scoped process" + + +asyncio.run(fix1_bridge_exec_contract()) +asyncio.run(fix2_workdir_probe_falls_back()) +asyncio.run(fix3_infra_failure_reclaims_scoped_processes()) +print("bridge-contract ok") +`; +} + function pythonAdapterSmokeScript(root: string): string { return String.raw` import json @@ -862,23 +1048,16 @@ assert "/proc/[0-9]*/environ" in term_cleanup, term_cleanup assert "kill -TERM" in term_cleanup, term_cleanup assert "kill -KILL" in kill_cleanup, kill_cleanup -class DelayedToolAgent: +class DelayedToolEnvironment: + # The bridged tool command now runs through environment.exec (a bare + # container exec), so the delayed-drain behaviour lives on the environment, + # not on the agent's exec_as_agent (which only handles scoped cleanup). def __init__(self): self.started = asyncio.Event() self.spawned = [] self.cancelled = False - def _cell_timeout_sec(self): - return 1 - - async def exec_as_agent(self, environment, command, **kwargs): - if "/proc/[0-9]*/environ" in command: - if "kill -KILL" in command: - for process in self.spawned: - if process.returncode is None: - process.kill() - await process.wait() - return types.SimpleNamespace(stdout="", stderr="", exit_code=0) + async def exec(self, command, cwd=None, timeout_sec=None, **kwargs): self.started.set() try: await asyncio.sleep(0.5) @@ -888,13 +1067,26 @@ class DelayedToolAgent: process = await asyncio.create_subprocess_exec("sleep", "30") self.spawned.append(process) await process.wait() - return types.SimpleNamespace(stdout="", stderr="", exit_code=0) + return types.SimpleNamespace(stdout="", stderr="", return_code=0) + +class DelayedToolCleanupAgent: + def __init__(self, environment): + self._environment = environment + + async def exec_as_agent(self, environment, command, **kwargs): + if "/proc/[0-9]*/environ" in command and "kill -KILL" in command: + for process in self._environment.spawned: + if process.returncode is None: + process.kill() + await process.wait() + return types.SimpleNamespace(stdout="", stderr="", return_code=0) async def assert_abnormal_teardown_drains_delayed_tool_start(): - agent = DelayedToolAgent() + environment = DelayedToolEnvironment() + agent = DelayedToolCleanupAgent(environment) request_thread = None try: - async with maka_agent_mod._ToolExecutorServer(agent, object()) as server: + async with maka_agent_mod._ToolExecutorServer(agent, environment) as server: def send_request(): request = urllib.request.Request( server.url + "/exec", @@ -909,14 +1101,14 @@ async def assert_abnormal_teardown_drains_delayed_tool_start(): request_thread = threading.Thread(target=send_request, daemon=True) request_thread.start() - await asyncio.wait_for(agent.started.wait(), timeout=1) + await asyncio.wait_for(environment.started.wait(), timeout=1) raise ValueError("cell failed") except ValueError: pass request_thread.join(timeout=1) - assert not agent.cancelled, "executor cancelled the Harbor Docker execution instead of draining it" - assert all(process.returncode is not None for process in agent.spawned), "tool command survived executor teardown" + assert not environment.cancelled, "executor cancelled the Harbor Docker execution instead of draining it" + assert all(process.returncode is not None for process in environment.spawned), "tool command survived executor teardown" asyncio.run(assert_abnormal_teardown_drains_delayed_tool_start()) From ada9837e70656add8892be67d87159fde5198d86 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 18:42:28 +0800 Subject: [PATCH 6/8] fix(headless): alias MAKA_AUTONOMOUS_MAX_ATTEMPTS and honor MAKA_BENCHMARK_DATASET override Map MAKA_AUTONOMOUS_MAX_ATTEMPTS onto MAKA_MAX_ATTEMPTS in the task-run env normalizer (the CLI path only reads MAKA_MAX_ATTEMPTS), alongside the existing MAKA_HARBOR_MAX_ATTEMPTS alias. Forward process.env MAKA_BENCHMARK_DATASET into the smoke job-config builder overrides so an explicit dataset value wins over the datasetName default, matching the retired shell runner. --- packages/headless/harbor/maka_agent.py | 2 ++ packages/headless/harbor/run-terminal-bench-smoke.mjs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/packages/headless/harbor/maka_agent.py b/packages/headless/harbor/maka_agent.py index 8949cbddde..37e37dff80 100644 --- a/packages/headless/harbor/maka_agent.py +++ b/packages/headless/harbor/maka_agent.py @@ -1201,6 +1201,8 @@ def _normalize_cli_env(env: dict[str, str]) -> None: env.setdefault("MAKA_STORAGE_ROOT", str(Path(env["MAKA_TASK_RUN_OUT_DIR"]) / "runs")) if env.get("MAKA_HARBOR_MAX_ATTEMPTS"): env.setdefault("MAKA_MAX_ATTEMPTS", env["MAKA_HARBOR_MAX_ATTEMPTS"]) + if env.get("MAKA_AUTONOMOUS_MAX_ATTEMPTS"): + env.setdefault("MAKA_MAX_ATTEMPTS", env["MAKA_AUTONOMOUS_MAX_ATTEMPTS"]) if env.get("MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"): env.setdefault("MAKA_MAX_RUNTIME_STEPS", env["MAKA_AUTONOMOUS_MAX_RUNTIME_STEPS"]) if env.get("MAKA_AUTONOMOUS_MAX_WALL_TIME_SEC"): diff --git a/packages/headless/harbor/run-terminal-bench-smoke.mjs b/packages/headless/harbor/run-terminal-bench-smoke.mjs index 4435c99e48..ff815c5303 100644 --- a/packages/headless/harbor/run-terminal-bench-smoke.mjs +++ b/packages/headless/harbor/run-terminal-bench-smoke.mjs @@ -116,6 +116,10 @@ function overridesFor(opts) { ...(opts.agentTimeoutSec !== undefined ? { agentTimeoutSec: opts.agentTimeoutSec } : {}), ...(opts.datasetName !== undefined ? { datasetName: opts.datasetName } : {}), ...(opts.datasetVersion !== undefined ? { datasetVersion: opts.datasetVersion } : {}), + // Match the retired shell runner: MAKA_BENCHMARK_DATASET in the environment + // overrides the dataset-name default that maka-* profiles forward to the + // adapter (an explicit value wins over the datasetName default). + ...(process.env.MAKA_BENCHMARK_DATASET ? { benchmarkDataset: process.env.MAKA_BENCHMARK_DATASET } : {}), }; } From 18fd88117acddca7583307755c3c70d5b4f916f8 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 18:42:35 +0800 Subject: [PATCH 7/8] refactor(headless): stop re-exporting harbor-smoke-config from the public barrel The only consumers are package-internal (the #harbor-smoke-config alias and relative test imports), so drop the buildSmokeJobConfig/resolveSmokeRunTargets value exports and the Smoke* type re-exports from src/index.ts. The public surface stays curated. --- packages/headless/src/index.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/headless/src/index.ts b/packages/headless/src/index.ts index 9bd099ac3c..c471a1ab08 100644 --- a/packages/headless/src/index.ts +++ b/packages/headless/src/index.ts @@ -4,20 +4,6 @@ // package-local entrypoints, not the root API. Minimal usage is // `runExperiment(config, task, { storageRoot })`. export { runPromptOptimizationRun } from './prompt-optimization-run.js'; -export { - buildSmokeJobConfig, - resolveSmokeRunTargets, -} from './harbor-smoke-config.js'; -export type { - SmokeManifest, - SmokeManifestAgent, - SmokeManifestDataset, - SmokeManifestDefaults, - SmokeManifestProfile, - SmokeConfigOverrides, - SmokeJobConfigResult, - SmokeRunTarget, -} from './harbor-smoke-config.js'; export type { MakaChangeAuditRecord } from './change-audit.js'; export type { PromptOptimizationRunInput, From 5643238e68a54400a0bf4947c512b9918c1c6978 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 16 Jul 2026 18:42:35 +0800 Subject: [PATCH 8/8] docs(headless): note continuation env vars are inert in task-run smoke profiles Append to the maka-prune-default / maka-stale-off / maka-retrieval-on profile descriptions that their continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing). Description fields do not enter the generated Harbor config, so fidelity diffs are unaffected. --- packages/headless/harbor/terminal-bench-smoke-profiles.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/headless/harbor/terminal-bench-smoke-profiles.json b/packages/headless/harbor/terminal-bench-smoke-profiles.json index 1db42fe937..30c0f9edef 100644 --- a/packages/headless/harbor/terminal-bench-smoke-profiles.json +++ b/packages/headless/harbor/terminal-bench-smoke-profiles.json @@ -78,7 +78,7 @@ } }, "maka-prune-default": { - "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B.", + "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).", "agentTimeoutMultiplier": 4.0, "agent": { "importPath": "maka_agent:MakaAgent", @@ -96,7 +96,7 @@ } }, "maka-stale-off": { - "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B.", + "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).", "agentTimeoutMultiplier": 4.0, "agent": { "importPath": "maka_agent:MakaAgent", @@ -115,7 +115,7 @@ } }, "maka-retrieval-on": { - "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B.", + "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).", "agentTimeoutMultiplier": 4.0, "agent": { "importPath": "maka_agent:MakaAgent",