refactor(headless): consolidate terminal-bench-smoke into packages/headless

packages/headless/.gitignore

@@ -0,0 +1,2 @@
+/harbor/smoke-jobs/
+/harbor/smoke-generated-configs/

packages/headless/README.md

@@ -161,6 +161,27 @@ trajectory/runtime refs, submitted snapshot metadata, verifier output, score,
 budget, isolation, permission/inbox facts, taxonomy, and warnings. They do not
 embed environment variables, credentials, or hidden harness configuration.
 
+## Terminal-Bench smoke runner
+
+`harbor/run-terminal-bench-smoke.mjs` is the local structured smoke harness for the
+`terminal-bench-sample` registry dataset. It reads the checked-in profile manifest
+`harbor/terminal-bench-smoke-profiles.json`, generates a Harbor run config under
+`harbor/smoke-generated-configs/`, and (unless `--dry-run`) invokes Harbor with the
+adapter directory on `PYTHONPATH`. `HARBOR_BIN` overrides the Harbor executable
+(default `harbor` on `PATH`).
+
+The `maka-*` profiles drive the single authoritative adapter `maka_agent:MakaAgent`
+in task-run host-bridge mode (`MAKA_HARBOR_MODE=task-run`): Maka runs the full
+task-run controller on the host and bridges tool execution into the task container,
+while the container installs nothing. `maka-heavy` and `maka-heavy-prune` carry the
+heavy-task and autonomous prior-attempt-replay experiments; `opencode` and `oracle`
+provide comparison and cheap dataset smoke arms.
+
+```sh
+node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-heavy --dry-run
+node packages/headless/harbor/run-terminal-bench-smoke.mjs --compare --task '*sqlite-with-gcov'
+```
+
 ## GLM-5.2 harness comparison
 
 `harbor/run-harness-ab.mjs` compares Maka and OpenCode 1.17.18 on the same Terminal-Bench 2.1 tasks with GLM-5.2 Max. The task root must match the 89 task ids and canonical task-tree fingerprint of the frozen official revision; a matching Harbor export with one task directory per id is accepted directly. Before model sampling, Harbor's Oracle inspects tasks in the frozen seeded order under the same verifier policy and selects the first 30 that pass. The immutable qualification evidence and selected task ids are bound into the run manifest. Maka keeps active and stale tool-result pruning enabled while semantic compact is explicitly disabled in both the manifest and runtime environment.

packages/headless/harbor/run-terminal-bench-smoke.mjs

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+
+/**
+ * Run a structured Terminal-Bench sample job through the local Harbor smoke
+ * harness. Replaces the retired terminal-bench-smoke/run-terminal-bench-sample.sh
+ * and run-terminal-bench-sample-heavy.sh shell scripts with a single pure-Node
+ * entrypoint. Maka profiles drive the authoritative maka_agent:MakaAgent adapter
+ * in task-run host-bridge mode; heavy-task and autonomous experiments run through
+ * `--profile maka-heavy` and `--profile maka-heavy-prune`.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { delimiter, dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildSmokeJobConfig, resolveSmokeRunTargets } from '#harbor-smoke-config';
+
+const HARBOR_DIR = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = resolve(HARBOR_DIR, '..', '..', '..');
+const MANIFEST_PATH = join(HARBOR_DIR, 'terminal-bench-smoke-profiles.json');
+
+const USAGE = `Run a structured Terminal-Bench sample job through the local Harbor smoke harness.
+
+Usage:
+  node packages/headless/harbor/run-terminal-bench-smoke.mjs [options]
+
+Profiles:
+  maka-basic         Maka task-run host bridge, non-autonomous, DeepSeek V4 Pro (default)
+  maka-heavy         Maka task-run heavy-task bridge for trace/evidence experiments
+  maka-heavy-prune   Maka heavy-task bridge with autonomous prior-attempt runtime replay
+                     and stale tool-result archive pruning enabled
+  maka-prune-default Post-#621 default prune pipeline with continuation (stale A/B B arm)
+  maka-stale-off     maka-prune-default with stale prune explicitly off (stale A/B A arm)
+  maka-retrieval-on  maka-prune-default plus eager archive retrieval (retrieval A/B B arm)
+  opencode           OpenCode Harbor wrapper
+  oracle             Harbor oracle agent for cheap wrapper/dataset smoke tests
+
+Options:
+  --profile NAME              Run profile (default: maka-basic)
+  --compare                   Run comparison profiles sequentially (default: maka-basic,opencode)
+  --compare-profiles LIST     Comma-separated profiles for --compare
+  --task PATTERN              Harbor task pattern (default: *sqlite-with-gcov)
+  --n-tasks N                 Pick N tasks instead of using --task
+  --job-name NAME             Harbor job name (default: generated with timestamp)
+  --model MODEL               Override model. For Maka this sets MAKA_MODEL; for OpenCode it sets model_name.
+  --steps N                   Override MAKA_MAX_STEPS for Maka profiles
+  --agent-timeout-sec N       Override MAKA_HARBOR_AGENT_TIMEOUT_SEC for Maka profiles
+  --dataset NAME              Override dataset name (default: terminal-bench-sample)
+  --dataset-version VERSION   Override dataset version (default: 2.0)
+  --dry-run                   Generate and print config path/command without running Harbor
+  -h, --help                  Show this help
+
+Environment:
+  HARBOR_BIN                  Harbor executable (default: harbor on PATH)
+
+Examples:
+  node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile oracle --n-tasks 1
+  node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-basic --task '*sqlite-with-gcov'
+  node packages/headless/harbor/run-terminal-bench-smoke.mjs --compare --task '*sqlite-with-gcov'
+  node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-heavy --compare-profiles maka-heavy,opencode --compare
+`;
+
+function parseArgs(argv) {
+  const opts = {
+    profile: 'maka-basic',
+    compare: false,
+    compareProfiles: 'maka-basic,opencode',
+    taskPattern: undefined,
+    nTasks: undefined,
+    jobName: undefined,
+    model: undefined,
+    maxSteps: undefined,
+    agentTimeoutSec: undefined,
+    datasetName: undefined,
+    datasetVersion: undefined,
+    dryRun: false,
+    help: false,
+  };
+  const takeValue = (i, flag) => {
+    const value = argv[i + 1];
+    if (value === undefined) {
+      throw new Error(`missing value for ${flag}`);
+    }
+    return value;
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    switch (arg) {
+      case '--profile': opts.profile = takeValue(i, arg); i++; break;
+      case '--compare': opts.compare = true; break;
+      case '--compare-profiles': opts.compare = true; opts.compareProfiles = takeValue(i, arg); i++; break;
+      case '--task': opts.taskPattern = takeValue(i, arg); i++; break;
+      case '--n-tasks': opts.nTasks = Number(takeValue(i, arg)); i++; break;
+      case '--job-name': opts.jobName = takeValue(i, arg); i++; break;
+      case '--model': opts.model = takeValue(i, arg); i++; break;
+      case '--steps': opts.maxSteps = takeValue(i, arg); i++; break;
+      case '--agent-timeout-sec': opts.agentTimeoutSec = takeValue(i, arg); i++; break;
+      case '--dataset': opts.datasetName = takeValue(i, arg); i++; break;
+      case '--dataset-version': opts.datasetVersion = takeValue(i, arg); i++; break;
+      case '--dry-run': opts.dryRun = true; break;
+      case '-h':
+      case '--help': opts.help = true; break;
+      default:
+        throw new Error(`unknown option: ${arg}`);
+    }
+  }
+  return opts;
+}
+
+function overridesFor(opts) {
+  return {
+    ...(opts.taskPattern !== undefined ? { taskPattern: opts.taskPattern } : {}),
+    ...(opts.nTasks !== undefined ? { nTasks: opts.nTasks } : {}),
+    ...(opts.model !== undefined ? { model: opts.model } : {}),
+    ...(opts.maxSteps !== undefined ? { maxSteps: opts.maxSteps } : {}),
+    ...(opts.agentTimeoutSec !== undefined ? { agentTimeoutSec: opts.agentTimeoutSec } : {}),
+    ...(opts.datasetName !== undefined ? { datasetName: opts.datasetName } : {}),
+    ...(opts.datasetVersion !== undefined ? { datasetVersion: opts.datasetVersion } : {}),
+    // Match the retired shell runner: MAKA_BENCHMARK_DATASET in the environment
+    // overrides the dataset-name default that maka-* profiles forward to the
+    // adapter (an explicit value wins over the datasetName default).
+    ...(process.env.MAKA_BENCHMARK_DATASET ? { benchmarkDataset: process.env.MAKA_BENCHMARK_DATASET } : {}),
+  };
+}
+
+function main() {
+  let opts;
+  try {
+    opts = parseArgs(process.argv.slice(2));
+  } catch (error) {
+    process.stderr.write(`${error.message}\n\n${USAGE}`);
+    process.exit(2);
+  }
+  if (opts.help) {
+    process.stdout.write(USAGE);
+    return;
+  }
+  if (opts.nTasks !== undefined && (!Number.isInteger(opts.nTasks) || opts.nTasks <= 0)) {
+    process.stderr.write(`--n-tasks must be a positive integer\n`);
+    process.exit(2);
+  }
+
+  const manifest = JSON.parse(readFileSync(MANIFEST_PATH, 'utf8'));
+  const generatedConfigDir = resolve(
+    REPO_ROOT,
+    manifest.defaults?.generatedConfigDir ?? 'packages/headless/harbor/smoke-generated-configs',
+  );
+  mkdirSync(generatedConfigDir, { recursive: true });
+
+  const harborBin = process.env.HARBOR_BIN || 'harbor';
+  const pythonPath = [HARBOR_DIR, process.env.PYTHONPATH].filter(Boolean).join(delimiter);
+
+  const targets = resolveSmokeRunTargets({
+    compare: opts.compare,
+    compareProfiles: opts.compareProfiles,
+    profile: opts.profile,
+    jobName: opts.jobName,
+  });
+
+  for (const target of targets) {
+    const { jobName, config } = buildSmokeJobConfig({
+      manifest,
+      profileName: target.profileName,
+      overrides: {
+        ...overridesFor(opts),
+        ...(target.jobName ? { jobName: target.jobName } : {}),
+      },
+    });
+    const configPath = join(generatedConfigDir, `${jobName}.json`);
+    writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
+
+    process.stdout.write(`Generated Harbor config: ${configPath}\n`);
+    process.stdout.write(`Profile: ${target.profileName}\n`);
+    process.stdout.write(`Run command:\n`);
+    process.stdout.write(`  PYTHONPATH=${HARBOR_DIR} ${harborBin} run --config ${configPath} --yes\n`);
+
+    if (opts.dryRun) continue;
+
+    const result = spawnSync(harborBin, ['run', '--config', configPath, '--yes'], {
+      cwd: REPO_ROOT,
+      stdio: 'inherit',
+      env: { ...process.env, PYTHONPATH: pythonPath },
+    });
+    if (result.error) {
+      process.stderr.write(`failed to launch harbor: ${result.error.message}\n`);
+      process.exit(1);
+    }
+    if (result.status !== 0) {
+      process.exit(result.status ?? 1);
+    }
+  }
+}
+
+main();

terminal-bench-smoke/terminal-bench-sample-runs.json → packages/headless/harbor/terminal-bench-smoke-profiles.json

@@ -1,9 +1,9 @@
 {
   "schemaVersion": 1,
-  "description": "Structured Terminal-Bench sample run profiles for the local Harbor smoke harness.",
+  "description": "Structured Terminal-Bench sample run profiles for the local Harbor smoke harness. Maka profiles drive the authoritative maka_agent:MakaAgent adapter in task-run host-bridge mode (MAKA_HARBOR_MODE=task-run).",
   "defaults": {
-    "jobsDir": "terminal-bench-smoke/jobs",
-    "generatedConfigDir": "terminal-bench-smoke/generated-configs",
+    "jobsDir": "packages/headless/harbor/smoke-jobs",
+    "generatedConfigDir": "packages/headless/harbor/smoke-generated-configs",
     "dataset": {
       "name": "terminal-bench-sample",
       "version": "2.0"
@@ -17,11 +17,12 @@
   },
   "profiles": {
     "maka-basic": {
-      "description": "Maka Harbor bridge against terminal-bench-sample, matching the successful sqlite-with-gcov DeepSeek V4 Pro sample run shape.",
+      "description": "Maka task-run host bridge against terminal-bench-sample, matching the successful sqlite-with-gcov DeepSeek V4 Pro sample run shape.",
       "agentTimeoutMultiplier": 4.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "0",
           "MAKA_HARBOR_AUTONOMOUS": "0",
           "MAKA_MODEL": "deepseek-v4-pro",
@@ -34,8 +35,9 @@
       "description": "Maka task-run heavy-task bridge for public sample trace/evidence experiments.",
       "agentTimeoutMultiplier": 8.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "1",
           "MAKA_HARBOR_AUTONOMOUS": "0",
           "MAKA_HEAVY_TASK_MODE": "1",
@@ -49,8 +51,9 @@
       "description": "Maka heavy-task bridge with autonomous prior-attempt runtime replay and stale tool-result archive pruning enabled.",
       "agentTimeoutMultiplier": 8.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "1",
           "MAKA_HARBOR_AUTONOMOUS": "1",
           "MAKA_HARBOR_REPLAY_PRIOR_ATTEMPT_RUNTIME_CONTEXT": "1",
@@ -75,11 +78,12 @@
       }
     },
     "maka-prune-default": {
-      "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B.",
+      "description": "Post-#621 default prune pipeline (active + stale on) with continuation enabled. Per-turn step cap deliberately low so tasks cross turn boundaries and stale prune fires. B arm for the stale-prune A/B; A arm for the retrieval A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).",
       "agentTimeoutMultiplier": 4.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "0",
           "MAKA_HARBOR_AUTONOMOUS": "0",
           "MAKA_HARBOR_CONTINUATION": "on",
@@ -92,11 +96,12 @@
       }
     },
     "maka-stale-off": {
-      "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B.",
+      "description": "Same as maka-prune-default but stale tool-result prune explicitly off. A arm for the stale-prune A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).",
       "agentTimeoutMultiplier": 4.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "0",
           "MAKA_HARBOR_AUTONOMOUS": "0",
           "MAKA_HARBOR_CONTINUATION": "on",
@@ -110,11 +115,12 @@
       }
     },
     "maka-retrieval-on": {
-      "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B.",
+      "description": "Same as maka-prune-default plus eager archive retrieval, so stale-pruned placeholders hydrate back (newest first, bounded). B arm for the retrieval A/B. Note: the continuation env vars are currently inert in task-run mode (never consumed by the CLI path; pre-existing).",
       "agentTimeoutMultiplier": 4.0,
       "agent": {
-        "importPath": "maka_harbor_agent:MakaHarborAgent",
+        "importPath": "maka_agent:MakaAgent",
         "env": {
+          "MAKA_HARBOR_MODE": "task-run",
           "MAKA_HARBOR_USE_TASK_RUN": "0",
           "MAKA_HARBOR_AUTONOMOUS": "0",
           "MAKA_HARBOR_CONTINUATION": "on",

packages/headless/package.json

@@ -35,6 +35,7 @@
     "#ab-manifest": "./dist/ab-manifest.js",
     "#prompt-ab-run": "./dist/prompt-ab-run.js",
     "#harbor-task-runner": "./dist/harbor-task-runner.js",
+    "#harbor-smoke-config": "./dist/harbor-smoke-config.js",
     "#provider-env": "./dist/provider-env.js",
     "#harbor-cell": "./dist/harbor-cell.js",
     "#opencode-toolchain": "./dist/opencode-toolchain.js",