diff --git a/.github/workflows/benchmark-harbor.yml b/.github/workflows/benchmark-harbor.yml index 6024f005754..a750ab6dc3e 100644 --- a/.github/workflows/benchmark-harbor.yml +++ b/.github/workflows/benchmark-harbor.yml @@ -5,10 +5,12 @@ on: branches: [main] paths: - "benchmarks/harbor-buzz-orchestra/**" + - "benchmarks/buzz-dataset/**" - ".github/workflows/benchmark-harbor.yml" pull_request: paths: - "benchmarks/harbor-buzz-orchestra/**" + - "benchmarks/buzz-dataset/**" - ".github/workflows/benchmark-harbor.yml" permissions: @@ -31,6 +33,9 @@ jobs: python -m pip install --disable-pip-version-check -e ".[dev]" pytest -q ruff check . + # The task verifiers live in the sibling benchmarks/buzz-dataset, so + # they need the harness config passed explicitly to stay linted. + ruff check --config pyproject.toml ../buzz-dataset - name: Test provisioner working-directory: benchmarks/harbor-buzz-orchestra/testbed run: | diff --git a/Justfile b/Justfile index ce8647cf77c..b12dfe9536e 100644 --- a/Justfile +++ b/Justfile @@ -996,6 +996,31 @@ benchmark *ARGS: uv run --project benchmarks/harbor-buzz-orchestra/testbed \ benchmarks/harbor-buzz-orchestra/scripts/benchmark.py {{ARGS}} +# Run the benchmark adapter + testbed gate exactly as CI does (pytest + ruff, pinned ruff from pyproject) +benchmark-check: + #!/usr/bin/env bash + set -euo pipefail + cd "{{justfile_directory()}}/benchmarks/harbor-buzz-orchestra" + # CI installs the dev extra with pip, so pyproject — not uv.lock — decides + # which ruff lints. Read the pin from there so this recipe cannot drift + # from the workflow (a floating specifier once meant CI failed on RUF100 + # while the locked local ruff passed). + ruff_pin="$(grep -oE 'ruff==[0-9.]+' pyproject.toml | head -1 | cut -d= -f3)" + for project in . testbed; do + ( + cd "$project" + echo "── harbor-buzz-orchestra/$project (ruff $ruff_pin)" + uv run --frozen pytest -q + uvx "ruff@$ruff_pin" check . + uvx "ruff@$ruff_pin" format --check . + ) + done + # The task verifiers live in the sibling benchmarks/buzz-dataset, so they + # need the harness config passed explicitly to stay linted. + echo "── buzz-dataset (ruff $ruff_pin)" + uvx "ruff@$ruff_pin" check --config pyproject.toml ../buzz-dataset + uvx "ruff@$ruff_pin" format --check --config pyproject.toml ../buzz-dataset + # Stop the benchmark Docker stack (state and channels are kept) benchmark-down: docker compose --project-name buzz-benchmark down diff --git a/benchmarks/buzz-dataset/README.md b/benchmarks/buzz-dataset/README.md new file mode 100644 index 00000000000..2909d0cd871 --- /dev/null +++ b/benchmarks/buzz-dataset/README.md @@ -0,0 +1,47 @@ +# buzz-dataset + +Harbor tasks that score **Buzz product behavior**, not just task correctness. +Each task poses an ordinary-looking question; what is graded is how the agent +answers it through Buzz — where the reply lands, who it notifies, what it was +willing to read. + +| Task | Behavior under test | +| --- | --- | +| [`reply-to-thread`](reply-to-thread) | Answers in the user's thread instead of as a new top-level message | +| [`user-mention`](user-mention) | Hands the turn back with an event-level `p`-tag mention of the requesting human | +| [`read-named-path-outside-workspace`](read-named-path-outside-workspace) | Reads a path the user named explicitly instead of refusing it as out of bounds | +| [`create-channel-invite-users`](create-channel-invite-users) | Creates a channel with the exact shape, TTL, and membership asked for | + +For `reply-to-thread` and `user-mention` the graded behavior is **deliberately +absent from `instruction.md`** — it has to come from `buzz-acp`'s production +base prompt. Read a task's own `README.md` before editing its instruction or +verifier. + +## Running + +These tasks need the [`harbor-buzz-orchestra`](../harbor-buzz-orchestra) +harness, which launches the real `buzz-acp` → `buzz-agent` → `buzz-dev-mcp` +stack inside the task container and exports the relay snapshot each verifier +grades. Plain `harbor run` against this directory will not work, and neither +will `harbor run -a oracle` (no `solution/solve.sh` is shipped — the Oracle +agent replaces the Buzz agent, so no relay trial is provisioned). + +From the repo root: + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/reply-to-thread \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +Pass `--path benchmarks/buzz-dataset` to run the whole suite. The default +condition is one solo agent on `gpt-5.6-luna` at `thinking_effort: medium`, +which needs `OPENAI_COMPAT_API_KEY`; see +[the harness README](../harbor-buzz-orchestra/README.md#buzz-native-tasks) for +the alternative Sonnet condition and the evidence-snapshot contract. + +The verifiers are covered by fixture tests that live with the harness, in +`../harbor-buzz-orchestra/tests/`. diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/README.md b/benchmarks/buzz-dataset/create-channel-invite-users/README.md new file mode 100644 index 00000000000..f473b9533fe --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/README.md @@ -0,0 +1,77 @@ +# create-channel-invite-users + +## What the agent does + +Creates a temporary private stream channel named `fix-pr-1234` with a one-hour +lifetime and invites an exact subset of a seeded directory: three named users as +members and two named bots with the `bot` role +([instruction.md](instruction.md)). + +Unlike the other tasks in this suite, the graded behavior **is** stated in the +instruction. What makes it hard is precision at scale: the provisioner seeds 50 +users (`benchmark-user-01`…`50`) and 10 bots (`benchmark-bot-01`…`10`), so the +agent has to resolve five specific names out of sixty look-alikes and invite +nobody else. + +## Environment + +`python:3.12-slim-bookworm`, no extra packages: the agent never runs in this +container's shell. `BuzzOrchestraAgent` launches the real `buzz-acp` / +`buzz-agent` stack against a dedicated relay, and the agent does all its work +through `buzz channels create` / `channels invite`. Agent timeout 300s. + +Directory identities are derived deterministically from the owner key +(`BuzzTrialProvisioner._stable_credential`) without persisting any secret, and +`_seed_directory` skips profiles already published — so reruns are idempotent +and pubkeys are stable across trials. + +## Verifier + +Reads the post-agent `/logs/artifacts/buzz-evidence.json` snapshot. The +snapshot's `observed_channels` come from the production CLI +(`channels search --exact --include-archived` plus `channels members`), so the +verifier grades the same view a user would see. Every dimension is +programmatic; `reward` is the conjunction of all of them. + +| Dimension | Type | Measures | +| --- | --- | --- | +| `evidence_complete` | programmatic | Snapshot is v1, names this task, and carries all 60 directory rows (50 users + 10 bots), the 5 resolvable targets, and exactly one orchestrator. Harness health, not agent skill — a 0 here means the provisioner or relay is suspect | +| `channel_created` | programmatic | Exactly one channel named `fix-pr-1234` exists | +| `channel_shape` | programmatic | `channel_type = stream`, `visibility = private`, not archived | +| `temporary_channel` | programmatic | `ttl_seconds == 3600` — "for one hour", read from the kind:39000 `ttl` tag surfaced by `channels search` | +| `exact_membership` | programmatic | Member pubkeys are exactly the owner plus the 5 targets — no extras, no duplicates | +| `expected_roles` | programmatic | The 3 users hold `member`, the 2 bots hold `bot`, the creator holds `owner` | + +## Layout + +``` +create-channel-invite-users/ +├── instruction.md # Prompt posted to the agent as the trial user +├── task.toml # Metadata, timeouts, 1 CPU / 1 GiB environment +├── environment/Dockerfile # Bare python image; the relay stack is uploaded +└── tests/ + ├── test.sh # Runs verify.py against the evidence snapshot + └── verify.py # Deterministic scorer (see table above) +``` + +To change the target set, edit `task_fixtures.TARGET_USERS` / `TARGET_BOTS`, +`instruction.md`, and the matching constants at the top of `tests/verify.py` — +all three must agree, and `evidence_complete` will fail loudly if the directory +size drifts from 60. + +## Running + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/create-channel-invite-users \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +`harbor run -a oracle` does **not** work here, and no `solution/solve.sh` is +shipped: the Oracle agent replaces `BuzzOrchestraAgent`, so no relay trial is +provisioned and no evidence snapshot is exported. The verifier is covered +instead by fixture tests in +`../harbor-buzz-orchestra/tests/test_create_channel_invite_users_verifier.py`. diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/environment/Dockerfile b/benchmarks/buzz-dataset/create-channel-invite-users/environment/Dockerfile new file mode 100644 index 00000000000..29f16f3c412 --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /app diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/instruction.md b/benchmarks/buzz-dataset/create-channel-invite-users/instruction.md new file mode 100644 index 00000000000..a6f4f794523 --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/instruction.md @@ -0,0 +1,15 @@ +Create a temporary private stream channel named `fix-pr-1234` for one hour. + +Invite these users as members: + +- `benchmark-user-07` +- `benchmark-user-19` +- `benchmark-user-42` + +Invite these bots with the bot role: + +- `benchmark-bot-03` +- `benchmark-bot-08` + +Do not invite any other users or bots. When finished, reply briefly with what +you created. diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/task.toml b/benchmarks/buzz-dataset/create-channel-invite-users/task.toml new file mode 100644 index 00000000000..b4ec4821f91 --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" + +[task] +name = "buzz-native/create-channel-invite-users" +description = "Create a temporary PR channel with an exact subset of users and bots." +authors = [{ name = "Buzz" }] +keywords = ["buzz-native", "channels", "membership", "cli"] + +[metadata] +difficulty = "medium" +category = "collaboration" +tags = ["channels", "membership", "cli"] + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 30.0 + +[environment] +network_mode = "public" +cpus = 1 +memory_mb = 1024 +storage_mb = 1024 diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/tests/test.sh b/benchmarks/buzz-dataset/create-channel-invite-users/tests/test.sh new file mode 100755 index 00000000000..be6819ba68a --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +python3 /tests/verify.py \ + --evidence /logs/artifacts/buzz-evidence.json \ + --reward /logs/verifier/reward.json \ + --details /logs/verifier/details.json diff --git a/benchmarks/buzz-dataset/create-channel-invite-users/tests/verify.py b/benchmarks/buzz-dataset/create-channel-invite-users/tests/verify.py new file mode 100755 index 00000000000..2d59c2a58f7 --- /dev/null +++ b/benchmarks/buzz-dataset/create-channel-invite-users/tests/verify.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Deterministic verifier for exact Buzz channel creation and membership.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +CHANNEL_NAME = "fix-pr-1234" +TARGET_USERS = {"benchmark-user-07", "benchmark-user-19", "benchmark-user-42"} +TARGET_BOTS = {"benchmark-bot-03", "benchmark-bot-08"} + + +def _zero_metrics() -> dict[str, float]: + return { + "reward": 0.0, + "channel_created": 0.0, + "channel_shape": 0.0, + "temporary_channel": 0.0, + "exact_membership": 0.0, + "expected_roles": 0.0, + "evidence_complete": 0.0, + } + + +def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]: + if not isinstance(evidence, dict): + return _zero_metrics(), {"error": "evidence root is not an object"} + + directory_rows = [ + row for row in evidence.get("directory", []) if isinstance(row, dict) + ] + directory = { + row.get("name"): row + for row in directory_rows + if isinstance(row.get("name"), str) + } + channels = [ + channel + for channel in evidence.get("observed_channels", []) + if isinstance(channel, dict) and channel.get("name") == CHANNEL_NAME + ] + channel = channels[0] if len(channels) == 1 else None + identities = ( + evidence.get("identities") + if isinstance(evidence.get("identities"), dict) + else {} + ) + orchestrators = [ + row + for row in identities.values() + if isinstance(row, dict) and row.get("role") == "orchestrator" + ] + owner_pubkey = orchestrators[0].get("pubkey") if len(orchestrators) == 1 else None + + expected_names = TARGET_USERS | TARGET_BOTS + expected_targets = { + directory[name]["pubkey"]: "bot" if name in TARGET_BOTS else "member" + for name in expected_names + if name in directory and isinstance(directory[name].get("pubkey"), str) + } + expected_members = ( + {owner_pubkey: "owner", **expected_targets} + if isinstance(owner_pubkey, str) + else expected_targets + ) + member_rows = ( + [row for row in channel.get("members", []) if isinstance(row, dict)] + if channel is not None + else [] + ) + actual_members = { + row.get("pubkey"): row.get("role") + for row in member_rows + if isinstance(row.get("pubkey"), str) + } + + evidence_complete = float( + evidence.get("schema_version") == 1 + and evidence.get("task_name") == "create-channel-invite-users" + and len(directory_rows) == 60 + and len(directory) == 60 + and sum(row.get("role") == "user" for row in directory_rows) == 50 + and sum(row.get("role") == "bot" for row in directory_rows) == 10 + and len(expected_targets) == 5 + and len(orchestrators) == 1 + ) + channel_created = float(channel is not None) + channel_shape = float( + channel is not None + and channel.get("channel_type") == "stream" + and channel.get("visibility") == "private" + and channel.get("archived") is False + ) + temporary_channel = float( + channel is not None and channel.get("ttl_seconds") == 3600 + ) + exact_membership = float( + len(member_rows) == len(actual_members) + and set(actual_members) == set(expected_members) + ) + expected_roles = float(actual_members == expected_members) + reward = float( + all( + metric == 1.0 + for metric in ( + evidence_complete, + channel_created, + channel_shape, + temporary_channel, + exact_membership, + expected_roles, + ) + ) + ) + metrics = { + "reward": reward, + "channel_created": channel_created, + "channel_shape": channel_shape, + "temporary_channel": temporary_channel, + "exact_membership": exact_membership, + "expected_roles": expected_roles, + "evidence_complete": evidence_complete, + } + details = { + "matching_channel_count": len(channels), + "channel_id": channel.get("channel_id") if channel is not None else None, + "expected_members": expected_members, + "actual_members": actual_members, + } + return metrics, details + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--reward", type=Path, required=True) + parser.add_argument("--details", type=Path, required=True) + args = parser.parse_args() + + try: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + metrics, details = score_evidence(evidence) + except (OSError, json.JSONDecodeError) as error: + metrics, details = _zero_metrics(), {"error": str(error)} + + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") + args.details.write_text( + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/README.md b/benchmarks/buzz-dataset/read-named-path-outside-workspace/README.md new file mode 100644 index 00000000000..63b24c29ddb --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/README.md @@ -0,0 +1,72 @@ +# read-named-path-outside-workspace + +## What the agent does + +Reads one file the user names explicitly by path — +`~/.claude/skills/context-health-check/SKILL.md` — and reports its `CHECK_ID` +and `ACTION` values ([instruction.md](instruction.md)). + +This is a **regression case**, not a capability test. The failure it guards +against is an agent that treats a user-named absolute path as out of bounds and +refuses (or proposes copying the file into the workspace first) instead of just +reading it. See block/buzz#6261. + +## Environment + +`python:3.12-slim-bookworm` with `HOME=/home/buzz`, so `~` in the instruction +resolves to the seeded skill directory. The Dockerfile generates the +`CHECK_ID` marker with `secrets.token_hex` **at image build time**, so the +expected value cannot be memorized across runs; the verifier reads the +answer back out of the same file rather than hardcoding it. Agent timeout 300s. + +## Verifier + +Reads the post-agent `/logs/artifacts/buzz-evidence.json` snapshot plus the +seeded `SKILL.md` (via `--skill-file`) for the expected values. + +| Dimension | Type | Measures | +| --- | --- | --- | +| `evidence_complete` | programmatic | Snapshot is v1, untruncated, names this task, and resolves the task event, channel, one orchestrator, and a candidate reply. Harness health, not agent skill | +| `expected_author` | programmatic | The scored message was published by the orchestrator | +| `same_channel` | programmatic | Reply carries the trial channel's `h` tag | +| `named_path_read` | programmatic | Reply contains the build-time `CHECK_ID` marker — proof the file was actually read | +| `action_reported` | programmatic | Reply contains the `ACTION` line, matched case-insensitively with whitespace collapsed and trailing punctuation stripped | +| `refusal_free` | programmatic | **Diagnostic only — not part of `reward`.** Flags refusal / workspace-boundary phrasing so failures are easy to classify | + +`reward` is the conjunction of every dimension *except* `refusal_free`: a +genuine refusal cannot produce the `CHECK_ID` or `ACTION` values, so the +substantive checks already fail it, and keeping the regex out of the reward +stops a hedged-but-correct answer from scoring 0 on phrasing. + +`instruction.md` also says "Do not search other directories". That constraint +is intentionally unscored — the snapshot holds relay messages, not the agent's +tool calls. + +## Layout + +``` +read-named-path-outside-workspace/ +├── instruction.md # Prompt posted to the agent as the trial user +├── task.toml # Metadata, timeouts, 1 CPU / 1 GiB environment +├── environment/Dockerfile # Seeds ~/.claude/skills/... with a random CHECK_ID +└── tests/ + ├── test.sh # Runs verify.py against the snapshot + SKILL.md + └── verify.py # Deterministic scorer (see table above) +``` + +## Running + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/read-named-path-outside-workspace \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +`harbor run -a oracle` does **not** work here, and no `solution/solve.sh` is +shipped: the Oracle agent replaces `BuzzOrchestraAgent`, so no relay trial is +provisioned and no evidence snapshot is exported. The verifier is covered +instead by fixture tests in +`../harbor-buzz-orchestra/tests/test_read_named_path_outside_workspace_verifier.py`. diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/environment/Dockerfile b/benchmarks/buzz-dataset/read-named-path-outside-workspace/environment/Dockerfile new file mode 100644 index 00000000000..3dd566334a6 --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/environment/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim-bookworm + +ENV HOME=/home/buzz +WORKDIR /app + +RUN mkdir -p /home/buzz/.claude/skills/context-health-check && \ + python -c "import secrets; from pathlib import Path; marker = 'chc-' + secrets.token_hex(12); Path('/home/buzz/.claude/skills/context-health-check/SKILL.md').write_text('# Context Health Check\n\nCHECK_ID: ' + marker + '\nACTION: Recommend compaction when remaining context is low.\n', encoding='utf-8')" diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/instruction.md b/benchmarks/buzz-dataset/read-named-path-outside-workspace/instruction.md new file mode 100644 index 00000000000..545fd2c5762 --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/instruction.md @@ -0,0 +1,4 @@ +Read the explicitly named file +`~/.claude/skills/context-health-check/SKILL.md`. Report its exact `CHECK_ID` +value and quote its `ACTION` value verbatim, in one concise sentence. Do not +search other directories. diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml b/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml new file mode 100644 index 00000000000..b4b61769153 --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" + +[task] +name = "buzz-native/read-named-path-outside-workspace" +description = "Read an explicitly named file outside the Buzz workspace." +authors = [{ name = "Buzz" }] +keywords = ["buzz-native", "filesystem", "workspace", "named-path"] + +[metadata] +difficulty = "easy" +category = "collaboration" +tags = ["filesystem", "named-path", "regression"] + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 30.0 + +[environment] +network_mode = "public" +cpus = 1 +memory_mb = 1024 +storage_mb = 1024 diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/test.sh b/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/test.sh new file mode 100755 index 00000000000..fed0b880caa --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +python3 /tests/verify.py \ + --evidence /logs/artifacts/buzz-evidence.json \ + --skill-file /home/buzz/.claude/skills/context-health-check/SKILL.md \ + --reward /logs/verifier/reward.json \ + --details /logs/verifier/details.json diff --git a/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/verify.py b/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/verify.py new file mode 100755 index 00000000000..889f6796049 --- /dev/null +++ b/benchmarks/buzz-dataset/read-named-path-outside-workspace/tests/verify.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Verifier for reading a user-named path outside the Buzz workspace.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +CHECK_ID = re.compile(r"^CHECK_ID:\s*(\S+)\s*$", re.MULTILINE) +ACTION = re.compile(r"^ACTION:\s*(\S.*\S|\S)\s*$", re.MULTILINE) +REFUSAL = re.compile( + r"\b(?:cannot|can't|unable to|won't|will not|refuse to|decline to)\s+" + r"(?:access|read|open|search)\b" + r"|workspace[- ]boundary" + r"|copy\s+.*\s+into\s+.*workspace", + re.IGNORECASE, +) + + +def _normalize(text: str) -> str: + """Fold case, collapse whitespace, drop trailing punctuation. + + The agent reports the ACTION line inside a sentence of its own, so line + wrapping and a dropped final period are presentation, not a wrong answer. + """ + return re.sub(r"\s+", " ", text).strip().strip(".!").casefold() + + +def _zero_metrics() -> dict[str, float]: + return { + "reward": 0.0, + "named_path_read": 0.0, + "action_reported": 0.0, + "refusal_free": 0.0, + "same_channel": 0.0, + "expected_author": 0.0, + "evidence_complete": 0.0, + } + + +def load_expectations(skill_file: Path) -> tuple[str, str]: + content = skill_file.read_text(encoding="utf-8") + check_id = CHECK_ID.search(content) + action = ACTION.search(content) + if check_id is None or action is None: + raise ValueError(f"fixture is missing CHECK_ID or ACTION: {skill_file}") + return check_id.group(1), action.group(1) + + +def score_evidence( + evidence: object, *, expected_check_id: str, expected_action: str +) -> tuple[dict[str, float], dict[str, Any]]: + if not isinstance(evidence, dict): + return _zero_metrics(), {"error": "evidence root is not an object"} + + task_event_id = evidence.get("task_event_id") + trial = evidence.get("trial") if isinstance(evidence.get("trial"), dict) else {} + channel_id = trial.get("channel_id") + identities = ( + evidence.get("identities") + if isinstance(evidence.get("identities"), dict) + else {} + ) + agents = [ + row + for row in identities.values() + if isinstance(row, dict) and row.get("role") == "orchestrator" + ] + agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None + messages = [ + message for message in evidence.get("messages", []) if isinstance(message, dict) + ] + root_indexes = [ + index + for index, message in enumerate(messages) + if message.get("id") == task_event_id + ] + root_index = root_indexes[0] if len(root_indexes) == 1 else -1 + candidates = [ + message + for message in messages[root_index + 1 :] + if agent_pubkey and message.get("pubkey") == agent_pubkey + ] + final = candidates[-1] if candidates else None + content = str(final.get("content", "")) if final is not None else "" + + evidence_complete = float( + evidence.get("schema_version") == 1 + and evidence.get("task_name") == "read-named-path-outside-workspace" + and evidence.get("truncated") is False + and isinstance(task_event_id, str) + and len(root_indexes) == 1 + and isinstance(channel_id, str) + and len(agents) == 1 + and final is not None + ) + expected_author = float(final is not None and final.get("pubkey") == agent_pubkey) + same_channel = float( + final is not None + and final.get("channel_id") == channel_id + and ["h", channel_id] in final.get("tags", []) + ) + named_path_read = float(expected_check_id in content) + action_reported = float(_normalize(expected_action) in _normalize(content)) + # Diagnostic only. A genuine refusal cannot report CHECK_ID or ACTION, so + # the two checks above already fail it; keeping this pattern out of the + # reward stops a hedged-but-correct answer from scoring 0 on a regex. + refusal_free = float(final is not None and REFUSAL.search(content) is None) + reward = float( + all( + metric == 1.0 + for metric in ( + evidence_complete, + expected_author, + same_channel, + named_path_read, + action_reported, + ) + ) + ) + metrics = { + "reward": reward, + "named_path_read": named_path_read, + "action_reported": action_reported, + "refusal_free": refusal_free, + "same_channel": same_channel, + "expected_author": expected_author, + "evidence_complete": evidence_complete, + } + details = { + "task_event_id": task_event_id, + "selected_message_id": final.get("id") if final is not None else None, + "selected_message_content": content if final is not None else None, + "expected_check_id": expected_check_id, + "expected_action": expected_action, + } + return metrics, details + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--skill-file", type=Path, required=True) + parser.add_argument("--reward", type=Path, required=True) + parser.add_argument("--details", type=Path, required=True) + args = parser.parse_args() + + try: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + expected_check_id, expected_action = load_expectations(args.skill_file) + metrics, details = score_evidence( + evidence, + expected_check_id=expected_check_id, + expected_action=expected_action, + ) + except (OSError, ValueError, json.JSONDecodeError) as error: + metrics, details = _zero_metrics(), {"error": str(error)} + + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") + args.details.write_text( + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/buzz-dataset/reply-to-thread/README.md b/benchmarks/buzz-dataset/reply-to-thread/README.md new file mode 100644 index 00000000000..48b1aa2abda --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/README.md @@ -0,0 +1,71 @@ +# reply-to-thread + +## What the agent does + +Answers a six-month financial projection posted by the trial user +([instruction.md](instruction.md)). The arithmetic is incidental — this task +measures **where** the answer lands: in the user's thread, not as a new +top-level channel message. + +> **The instruction deliberately says nothing about threading.** Threading is +> the behavior under test, and it must come from `buzz-acp`'s production base +> prompt rather than from the task prompt. Do not "fix" the instruction by +> telling the agent to reply in-thread — that would make the task measure +> instruction-following instead of product behavior. + +## Environment + +`python:3.12-slim-bookworm`, no extra packages: the agent never runs in this +container's shell. `BuzzOrchestraAgent` launches the real `buzz-acp` / +`buzz-agent` stack against a dedicated relay, and the agent works entirely +through Buzz. Agent timeout 300s; the manifest's `trial_budget` is the +effective clock. + +## Verifier + +Reads the post-agent `/logs/artifacts/buzz-evidence.json` snapshot (written by +`BuzzContainerRuntime._collect_evidence` after the agent stops, so the agent +cannot influence it). Every dimension is programmatic; `reward` is the +conjunction of all of them. + +| Dimension | Type | Measures | +| --- | --- | --- | +| `evidence_complete` | programmatic | Snapshot is v1, untruncated, has one orchestrator, and resolves the task event and a candidate reply. Harness health, not agent skill — a 0 here means investigate the run | +| `expected_author` | programmatic | The scored message was published by the orchestrator | +| `same_channel` | programmatic | Reply carries the trial channel's `h` tag | +| `reply_to_thread` | programmatic | Reply carries `["e", , "", "reply"]` — the behavior under test | +| `answer_correct` | programmatic | Month-6 revenue (160,811), month-6 expenses (84,462), and cumulative profit (374,470), each ±1 and each on a line naming it | + +`answer_correct` requires the label and the value on the same line so a +work-showing table with a wrong stated answer cannot pass on its intermediate +rows. `instruction.md` asks for that formatting explicitly. + +## Layout + +``` +reply-to-thread/ +├── instruction.md # Prompt posted to the agent as the trial user +├── task.toml # Metadata, timeouts, 1 CPU / 1 GiB environment +├── environment/Dockerfile # Bare python image; the relay stack is uploaded +└── tests/ + ├── test.sh # Runs verify.py against the evidence snapshot + └── verify.py # Deterministic scorer (see table above) +``` + +## Running + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/reply-to-thread \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +`harbor run -a oracle` does **not** work here, and no `solution/solve.sh` is +shipped: the Oracle agent replaces `BuzzOrchestraAgent`, so no relay trial is +provisioned and no evidence snapshot is exported. The verifier is covered +instead by positive and negative fixture tests in +`../harbor-buzz-orchestra/tests/test_reply_to_thread_verifier.py` (run from +`benchmarks/harbor-buzz-orchestra`: `uv run --extra dev pytest -q`). diff --git a/benchmarks/buzz-dataset/reply-to-thread/environment/Dockerfile b/benchmarks/buzz-dataset/reply-to-thread/environment/Dockerfile new file mode 100644 index 00000000000..29f16f3c412 --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /app diff --git a/benchmarks/buzz-dataset/reply-to-thread/instruction.md b/benchmarks/buzz-dataset/reply-to-thread/instruction.md new file mode 100644 index 00000000000..64067c529ee --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/instruction.md @@ -0,0 +1,10 @@ +Prepare a six-month financial projection using these assumptions: + +- Month 0 revenue is $120,000 and expenses are $75,000. +- Revenue grows by 5% each month. +- Expenses grow by 2% each month. +- Project months 1 through 6, compounding each month from the prior month. + +Reply with the month 6 revenue, month 6 expenses, and cumulative operating +profit across months 1 through 6. Round each result to the nearest dollar, and +label each figure on the same line as its value. diff --git a/benchmarks/buzz-dataset/reply-to-thread/task.toml b/benchmarks/buzz-dataset/reply-to-thread/task.toml new file mode 100644 index 00000000000..9d400aabb7f --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" + +[task] +name = "buzz-native/reply-to-thread" +description = "Answer a financial projection in the thread started by the user." +authors = [{ name = "Buzz" }] +keywords = ["buzz-native", "messaging", "threading"] + +[metadata] +difficulty = "easy" +category = "collaboration" +tags = ["messaging", "threading", "implicit-behavior"] + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 30.0 + +[environment] +network_mode = "public" +cpus = 1 +memory_mb = 1024 +storage_mb = 1024 diff --git a/benchmarks/buzz-dataset/reply-to-thread/tests/test.sh b/benchmarks/buzz-dataset/reply-to-thread/tests/test.sh new file mode 100755 index 00000000000..3dd50814b6e --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +mkdir -p /logs/verifier +python3 /tests/verify.py \ + --evidence /logs/artifacts/buzz-evidence.json \ + --reward /logs/verifier/reward.json \ + --details /logs/verifier/details.json diff --git a/benchmarks/buzz-dataset/reply-to-thread/tests/verify.py b/benchmarks/buzz-dataset/reply-to-thread/tests/verify.py new file mode 100755 index 00000000000..cfe22109c10 --- /dev/null +++ b/benchmarks/buzz-dataset/reply-to-thread/tests/verify.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Deterministic verifier for the Buzz reply-to-thread task.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +# Each amount must appear on a line that also names what it is. Scanning the +# whole reply for bare numbers passes a work-showing table whose month-6 rows +# are right but whose stated answer is wrong. +EXPECTED_ANSWERS = ( + ("revenue", 160_811.0), + ("expense", 84_462.0), + ("profit", 374_470.0), +) +EXPECTED_AMOUNTS = tuple(amount for _, amount in EXPECTED_ANSWERS) +NUMBER = re.compile(r"(? list[float]: + values: list[float] = [] + for token in NUMBER.findall(content): + try: + values.append(float(token.replace("$", "").replace(",", ""))) + except ValueError: + continue + return values + + +def _contains_amount(values: list[float], expected: float) -> bool: + return any(abs(value - expected) <= 1.0 for value in values) + + +def _labelled_amount(content: str, label: str, expected: float) -> bool: + """Whether some line names ``label`` and carries ``expected`` on it.""" + return any( + label in line.casefold() and _contains_amount(_numbers(line), expected) + for line in content.splitlines() + ) + + +def _has_tag(message: dict[str, Any], expected: list[str]) -> bool: + return any(tag == expected for tag in message.get("tags", [])) + + +def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]: + if not isinstance(evidence, dict): + return _zero_metrics(), {"error": "evidence root is not an object"} + + task_event_id = evidence.get("task_event_id") + trial = evidence.get("trial") if isinstance(evidence.get("trial"), dict) else {} + channel_id = trial.get("channel_id") + identities = ( + evidence.get("identities") + if isinstance(evidence.get("identities"), dict) + else {} + ) + agents = [ + identity + for identity in identities.values() + if isinstance(identity, dict) and identity.get("role") == "orchestrator" + ] + agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None + messages = [ + message for message in evidence.get("messages", []) if isinstance(message, dict) + ] + root_indexes = [ + index + for index, message in enumerate(messages) + if message.get("id") == task_event_id + ] + root_index = root_indexes[0] if len(root_indexes) == 1 else -1 + candidates = [ + message + for message in messages[root_index + 1 :] + if agent_pubkey and message.get("pubkey") == agent_pubkey + ] + final = candidates[-1] if candidates else None + + evidence_complete = float( + evidence.get("schema_version") == 1 + and evidence.get("truncated") is False + and isinstance(task_event_id, str) + and len(root_indexes) == 1 + and isinstance(channel_id, str) + and len(agents) == 1 + and final is not None + ) + expected_author = float(final is not None and final.get("pubkey") == agent_pubkey) + same_channel = float( + final is not None + and final.get("channel_id") == channel_id + and _has_tag(final, ["h", channel_id]) + ) + reply_to_thread = float( + final is not None + and final.get("reply_to_event_id") == task_event_id + and _has_tag(final, ["e", task_event_id, "", "reply"]) + ) + content = str(final.get("content", "")) if final is not None else "" + values = _numbers(content) + answer_correct = float( + all( + _labelled_amount(content, label, expected) + for label, expected in EXPECTED_ANSWERS + ) + ) + reward = float( + all( + metric == 1.0 + for metric in ( + evidence_complete, + expected_author, + same_channel, + reply_to_thread, + answer_correct, + ) + ) + ) + metrics = { + "reward": reward, + "answer_correct": answer_correct, + "reply_to_thread": reply_to_thread, + "same_channel": same_channel, + "expected_author": expected_author, + "evidence_complete": evidence_complete, + } + details = { + "task_event_id": task_event_id, + "selected_message_id": final.get("id") if final is not None else None, + "selected_message_content": final.get("content") if final is not None else None, + "parsed_numbers": values, + "expected_amounts": list(EXPECTED_AMOUNTS), + } + return metrics, details + + +def _zero_metrics() -> dict[str, float]: + return { + "reward": 0.0, + "answer_correct": 0.0, + "reply_to_thread": 0.0, + "same_channel": 0.0, + "expected_author": 0.0, + "evidence_complete": 0.0, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--reward", type=Path, required=True) + parser.add_argument("--details", type=Path, required=True) + args = parser.parse_args() + + try: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + metrics, details = score_evidence(evidence) + except (OSError, json.JSONDecodeError) as error: + metrics, details = _zero_metrics(), {"error": str(error)} + + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") + args.details.write_text( + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/buzz-dataset/user-mention/README.md b/benchmarks/buzz-dataset/user-mention/README.md new file mode 100644 index 00000000000..e1bad8b4504 --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/README.md @@ -0,0 +1,66 @@ +# user-mention + +## What the agent does + +Answers a one-line licensing calculation ([instruction.md](instruction.md)). +The arithmetic is incidental — this task measures whether the agent hands the +turn back with an **event-level mention** of the requesting human, so the user +gets a real Buzz notification instead of a message they have to notice. + +> **The instruction deliberately says nothing about mentioning anyone.** The +> mention is the behavior under test and must come from `buzz-acp`'s production +> base prompt. Do not add "mention the user" to the instruction. + +The trial user for this task is provisioned with the stable three-word display +name `John Vincent Doe` (`task_fixtures.USER_MENTION_DISPLAY_NAME`), which +forces the agent to resolve a multi-word identity to a pubkey rather than +guessing a single-token handle. + +## Environment + +`python:3.12-slim-bookworm`, no extra packages: the agent never runs in this +container's shell. `BuzzOrchestraAgent` launches the real `buzz-acp` / +`buzz-agent` stack against a dedicated relay. Agent timeout 300s. + +## Verifier + +Reads the post-agent `/logs/artifacts/buzz-evidence.json` snapshot. Every +dimension is programmatic; `reward` is the conjunction of all of them. + +| Dimension | Type | Measures | +| --- | --- | --- | +| `evidence_complete` | programmatic | Snapshot is v1, untruncated, names this task, and resolves exactly one orchestrator, one user, and a candidate reply. Harness health, not agent skill | +| `three_word_user` | programmatic | The provisioner seeded the three-word display name. Fixture self-check | +| `expected_author` | programmatic | The scored message was published by the orchestrator | +| `same_channel` | programmatic | Reply carries the trial channel's `h` tag | +| `user_p_tagged` | programmatic | Reply carries a `p` tag for the user's pubkey — the behavior under test. Presentation-only `@text` does not count | +| `answer_correct` | programmatic | Annual total 5,328 (12 licenses × $37 × 12 months), ±1 | + +## Layout + +``` +user-mention/ +├── instruction.md # Prompt posted to the agent as the trial user +├── task.toml # Metadata, timeouts, 1 CPU / 1 GiB environment +├── environment/Dockerfile # Bare python image; the relay stack is uploaded +└── tests/ + ├── test.sh # Runs verify.py against the evidence snapshot + └── verify.py # Deterministic scorer (see table above) +``` + +## Running + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/user-mention \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +`harbor run -a oracle` does **not** work here, and no `solution/solve.sh` is +shipped: the Oracle agent replaces `BuzzOrchestraAgent`, so no relay trial is +provisioned and no evidence snapshot is exported. The verifier is covered +instead by fixture tests in +`../harbor-buzz-orchestra/tests/test_user_mention_verifier.py`. diff --git a/benchmarks/buzz-dataset/user-mention/environment/Dockerfile b/benchmarks/buzz-dataset/user-mention/environment/Dockerfile new file mode 100644 index 00000000000..29f16f3c412 --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /app diff --git a/benchmarks/buzz-dataset/user-mention/instruction.md b/benchmarks/buzz-dataset/user-mention/instruction.md new file mode 100644 index 00000000000..57048560537 --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/instruction.md @@ -0,0 +1,2 @@ +Calculate the annual cost of 12 software licenses priced at $37 per license +per month. Reply with the annual total in one concise sentence. diff --git a/benchmarks/buzz-dataset/user-mention/task.toml b/benchmarks/buzz-dataset/user-mention/task.toml new file mode 100644 index 00000000000..53659e146ba --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" + +[task] +name = "buzz-native/user-mention" +description = "Answer a calculation and mention the three-word user identity." +authors = [{ name = "Buzz" }] +keywords = ["buzz-native", "messaging", "mentions"] + +[metadata] +difficulty = "easy" +category = "collaboration" +tags = ["messaging", "mentions", "implicit-behavior"] + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 30.0 + +[environment] +network_mode = "public" +cpus = 1 +memory_mb = 1024 +storage_mb = 1024 diff --git a/benchmarks/buzz-dataset/user-mention/tests/test.sh b/benchmarks/buzz-dataset/user-mention/tests/test.sh new file mode 100755 index 00000000000..be6819ba68a --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +python3 /tests/verify.py \ + --evidence /logs/artifacts/buzz-evidence.json \ + --reward /logs/verifier/reward.json \ + --details /logs/verifier/details.json diff --git a/benchmarks/buzz-dataset/user-mention/tests/verify.py b/benchmarks/buzz-dataset/user-mention/tests/verify.py new file mode 100755 index 00000000000..68ca746f745 --- /dev/null +++ b/benchmarks/buzz-dataset/user-mention/tests/verify.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Deterministic verifier for a human-facing callback mention.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +USER_DISPLAY_NAME = "John Vincent Doe" +EXPECTED_TOTAL = 5_328.0 +NUMBER = re.compile(r"(? list[float]: + values: list[float] = [] + for token in NUMBER.findall(content): + try: + values.append(float(token.replace("$", "").replace(",", ""))) + except ValueError: + continue + return values + + +def _has_p_tag(message: dict[str, Any], pubkey: object) -> bool: + return isinstance(pubkey, str) and any( + isinstance(tag, list) and len(tag) >= 2 and tag[0] == "p" and tag[1] == pubkey + for tag in message.get("tags", []) + ) + + +def _zero_metrics() -> dict[str, float]: + return { + "reward": 0.0, + "answer_correct": 0.0, + "user_p_tagged": 0.0, + "three_word_user": 0.0, + "same_channel": 0.0, + "expected_author": 0.0, + "evidence_complete": 0.0, + } + + +def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]: + if not isinstance(evidence, dict): + return _zero_metrics(), {"error": "evidence root is not an object"} + + task_event_id = evidence.get("task_event_id") + trial = evidence.get("trial") if isinstance(evidence.get("trial"), dict) else {} + channel_id = trial.get("channel_id") + identities = ( + evidence.get("identities") + if isinstance(evidence.get("identities"), dict) + else {} + ) + agents = [ + row + for row in identities.values() + if isinstance(row, dict) and row.get("role") == "orchestrator" + ] + users = [ + (name, row) + for name, row in identities.items() + if isinstance(row, dict) and row.get("role") == "user" + ] + agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None + user_name, user = users[0] if len(users) == 1 else (None, {}) + user_pubkey = user.get("pubkey") + messages = [ + message for message in evidence.get("messages", []) if isinstance(message, dict) + ] + root_indexes = [ + index + for index, message in enumerate(messages) + if message.get("id") == task_event_id + ] + root_index = root_indexes[0] if len(root_indexes) == 1 else -1 + candidates = [ + message + for message in messages[root_index + 1 :] + if agent_pubkey and message.get("pubkey") == agent_pubkey + ] + final = candidates[-1] if candidates else None + content = str(final.get("content", "")) if final is not None else "" + values = _numbers(content) + + three_word_user = float( + user_name == USER_DISPLAY_NAME and len(USER_DISPLAY_NAME.split()) == 3 + ) + evidence_complete = float( + evidence.get("schema_version") == 1 + and evidence.get("task_name") == "user-mention" + and evidence.get("truncated") is False + and isinstance(task_event_id, str) + and len(root_indexes) == 1 + and isinstance(channel_id, str) + and len(agents) == 1 + and len(users) == 1 + and final is not None + ) + expected_author = float(final is not None and final.get("pubkey") == agent_pubkey) + same_channel = float( + final is not None + and final.get("channel_id") == channel_id + and ["h", channel_id] in final.get("tags", []) + ) + user_p_tagged = float( + final is not None + and _has_p_tag(final, user_pubkey) + and user_pubkey in final.get("mentioned_pubkeys", []) + ) + answer_correct = float(any(abs(value - EXPECTED_TOTAL) <= 1.0 for value in values)) + reward = float( + all( + metric == 1.0 + for metric in ( + evidence_complete, + expected_author, + same_channel, + three_word_user, + user_p_tagged, + answer_correct, + ) + ) + ) + metrics = { + "reward": reward, + "answer_correct": answer_correct, + "user_p_tagged": user_p_tagged, + "three_word_user": three_word_user, + "same_channel": same_channel, + "expected_author": expected_author, + "evidence_complete": evidence_complete, + } + details = { + "task_event_id": task_event_id, + "selected_message_id": final.get("id") if final is not None else None, + "selected_message_content": content if final is not None else None, + "user_display_name": user_name, + "user_pubkey": user_pubkey, + "parsed_numbers": values, + "expected_total": EXPECTED_TOTAL, + } + return metrics, details + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--reward", type=Path, required=True) + parser.add_argument("--details", type=Path, required=True) + args = parser.parse_args() + + try: + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + metrics, details = score_evidence(evidence) + except (OSError, json.JSONDecodeError) as error: + metrics, details = _zero_metrics(), {"error": str(error)} + + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") + args.details.write_text( + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor-buzz-orchestra/README.md b/benchmarks/harbor-buzz-orchestra/README.md index 0358c954e7d..b7df4a55579 100644 --- a/benchmarks/harbor-buzz-orchestra/README.md +++ b/benchmarks/harbor-buzz-orchestra/README.md @@ -62,6 +62,57 @@ rather than deletes that channel, leaving the relay/Postgres event timeline and the per-agent acp/agent logs (downloaded into the trial's `buzz/` artifacts) available for analysis. +### Buzz-native tasks + +The local [`benchmarks/buzz-dataset`](../buzz-dataset) suite — a sibling +directory of this harness, not a subdirectory of it — scores Buzz product +behavior alongside task correctness. It currently covers direct thread replies, callback user +mentions, targeted reads of user-named paths outside the workspace, and exact +channel creation/membership. Run one task with the production base prompt from +the checked-out source build: + +```bash +just benchmark \ + --path benchmarks/buzz-dataset/reply-to-thread \ + --attempts 1 \ + --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml \ + --endpoint-config benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json \ + --n-concurrent 1 +``` + +The default condition is `buzz-native-solo-luna.yaml` — one solo agent on +`gpt-5.6-luna` at `thinking_effort: medium`. What this suite scores comes from +the base prompt rather than from model strength, so the cheap model at a +middling effort is the right yardstick: a weak result here is a prompt finding, +not a model finding. It needs `OPENAI_COMPAT_API_KEY` and the explicit +`--endpoint-config` above, because `--endpoint-config` defaults to +`anthropic-live.json`. Swap in `buzz-native-solo-sonnet.yaml` (no +`--endpoint-config`, needs `ANTHROPIC_API_KEY`) to compare against Sonnet 4.6. + +A roster entry that does not pin `generation.thinking_effort` runs at the +runtime default (`THINKING_EFFORT`, currently `medium`) rather than at whatever +the provider defaults to, so the level is always recorded. Leaving it unset +does not change a condition's hash — manifests written before the effort axis +existed keep their identity and stay comparable to their earlier receipts. + +Replace the path with `benchmarks/buzz-dataset/create-channel-invite-users` +to run the channel task. Its provisioner seeds a stable directory of 50 users +and 10 bots, while the verifier checks the created channel's TTL and exact +membership through post-agent CLI evidence. + +After the agent stops, the runtime snapshots public relay state (source +messages plus any task-declared channels and members) to +`/logs/artifacts/buzz-evidence.json`. The task verifier reads that post-agent +artifact; relay credentials and database access are never exposed to the model +or verifier. If the snapshot cannot be exported the trial **fails** rather than +scoring 0 — a harness fault and a model fault stay distinguishable — and the +cause is written to the trial's `buzz/buzz-evidence-error.txt`. + +Each task ships its own `README.md` documenting its reward dimensions and, for +the tasks whose graded Buzz behavior is deliberately absent from +`instruction.md` (`reply-to-thread`, `user-mention`), why that omission is the +point. Read it before editing a task's instruction or verifier. + ## Leaderboard runs `just benchmark` is the one-command path: it stands up a dedicated Docker diff --git a/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml b/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml new file mode 100644 index 00000000000..fe25d0e8eb2 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-luna.yaml @@ -0,0 +1,45 @@ +# Default condition for the local benchmarks/buzz-dataset suite: one production +# Buzz agent on gpt-5.6-luna at reasoning effort `medium`. +# +# buzz-acp supplies crates/buzz-acp/src/base_prompt.md from the checked-out +# source build; the persona below only establishes that there is no team. The +# suite scores Buzz product behavior (threading, mentions, exact membership), +# and that behavior comes from the base prompt — so the cheap model at a +# middling effort is the right default. A weak result here is a prompt finding, +# not a model finding. +# +# The endpoint name is the literal OpenAI model id: the runtime passes it to +# the provider as BUZZ_AGENT_MODEL. Resolution to provider/key lives in +# testbed/endpoints/openai-live.json (OPENAI_COMPAT_API_KEY), which is +# deployment config and deliberately outside this manifest — pass it with +# `--endpoint-config`, since the default is anthropic-live.json. +schema_version: "1" +condition: buzz-native-solo-luna-medium +roster: + - id: solo + kind: orchestrator + role: solo + count: 1 + endpoint: gpt-5.6-luna + model_revision: gpt-5.6-luna + prompt: + path: personas/buzz-native-solo.md + sha256: 972950f0e2bfb9bf540c98e70e075479ab80cd596cb5ad405dad0cafdc60840b + generation: + max_output_tokens: 4096 + context_window_tokens: 200000 + # Pinned rather than left implicit even though `medium` is also the + # runtime default, so the condition records the level it ran at. + thinking_effort: medium +prices: + # Repriced 2026-07-30 (luna 1.0/0.1/6.0 -> 0.20/0.02/1.20). Receipts from + # before that date carry the old rates; re-price measured tokens rather than + # editing a receipt. + gpt-5.6-luna: + input_per_million_usd: 0.2 + cached_input_per_million_usd: 0.02 + output_per_million_usd: 1.2 +trial_budget: + # Matches the tasks' own 300s agent timeout: these are single-turn + # collaboration checks, not long autonomous runs. + timeout_seconds: 300 diff --git a/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-sonnet.yaml b/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-sonnet.yaml new file mode 100644 index 00000000000..2948cead2f9 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-sonnet.yaml @@ -0,0 +1,25 @@ +# Single production Buzz agent for the local Buzz-native Harbor dataset. +# buzz-acp supplies crates/buzz-acp/src/base_prompt.md from the checked-out +# source build; this small persona only establishes that there is no team. +schema_version: "1" +condition: buzz-native-solo-sonnet46 +roster: + - id: solo + kind: orchestrator + role: solo + count: 1 + endpoint: claude-sonnet-4-6 + model_revision: claude-sonnet-4-6 + prompt: + path: personas/buzz-native-solo.md + sha256: 972950f0e2bfb9bf540c98e70e075479ab80cd596cb5ad405dad0cafdc60840b + generation: + max_output_tokens: 4096 + context_window_tokens: 200000 +prices: + claude-sonnet-4-6: + input_per_million_usd: 3 + cached_input_per_million_usd: 0.3 + output_per_million_usd: 15 +trial_budget: + timeout_seconds: 300 diff --git a/benchmarks/harbor-buzz-orchestra/personas/buzz-native-solo.md b/benchmarks/harbor-buzz-orchestra/personas/buzz-native-solo.md new file mode 100644 index 00000000000..1fc4820dd0e --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/personas/buzz-native-solo.md @@ -0,0 +1,2 @@ +You are the only agent assigned to this channel. Handle the user's request +directly and completely. Be very concise and direct. Use plain, simple language. diff --git a/benchmarks/harbor-buzz-orchestra/pyproject.toml b/benchmarks/harbor-buzz-orchestra/pyproject.toml index 1dd28780408..4896d053ba4 100644 --- a/benchmarks/harbor-buzz-orchestra/pyproject.toml +++ b/benchmarks/harbor-buzz-orchestra/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "hatchling.build" dev = [ "pytest>=8.4", "pytest-asyncio>=1.2", - "ruff>=0.15", + "ruff==0.16.3", ] [tool.pytest.ini_options] diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py index 1b79d233b9e..47766d4cd2c 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py @@ -7,13 +7,19 @@ RuntimeLaunchError, ) from .manifest import ExperimentManifest, ManifestError -from .provisioning import AgentCredential, TrialHandle, TrialProvisioner +from .provisioning import ( + AgentCredential, + DirectoryIdentity, + TrialHandle, + TrialProvisioner, +) from .runtime import OrchestraRuntime, RuntimeResult __all__ = [ "AgentCredential", "BuzzContainerRuntime", "BuzzOrchestraAgent", + "DirectoryIdentity", "EndpointLaunchConfig", "ExperimentManifest", "ManifestError", diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py index 3d1c81364f5..f98c8f4965f 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py @@ -170,7 +170,11 @@ async def run( # GUI shows one recognisable channel per problem per attempt. channel_label = getattr(environment, "environment_name", None) handle = self.provisioner.create_trial( - run_id, trial_id, self.manifest, channel_label=channel_label + run_id, + trial_id, + self.manifest, + channel_label=channel_label, + task_name=channel_label, ) if handle.trial_id != trial_id: raise RuntimeError("provisioner returned a handle for a different trial_id") diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index a0602111d13..756cbfa31f8 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -14,22 +14,33 @@ import json import os import shlex +import traceback from dataclasses import dataclass, field from pathlib import Path from typing import Any from harbor.environments.base import BaseEnvironment +from .evidence import build_buzz_evidence from .manifest import AgentClass, ExperimentManifest from .provisioning import AgentCredential, TrialHandle from .runtime import RuntimeResult - -DEFAULT_MAX_AGENT_ROUNDS = 0 # 0 = unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial budget is the clock +from .task_fixtures import fixture_for + +DEFAULT_MAX_AGENT_ROUNDS = ( + 0 # 0 = unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial budget is the clock +) +# Reasoning effort for a roster entry that does not pin one. Pinned here rather +# than left to the provider so an unset effort still means a recorded, stable +# level across endpoints instead of "whatever the provider happens to default +# to", which is neither captured in the condition hash nor comparable. +THINKING_EFFORT = "medium" # Container-side layout for the uploaded Buzz stack. REMOTE_ROOT = "/opt/buzz" REMOTE_BIN = f"{REMOTE_ROOT}/bin" REMOTE_PROMPTS = f"{REMOTE_ROOT}/prompts" REMOTE_LOGS = f"{REMOTE_ROOT}/logs" +REMOTE_EVIDENCE = "/logs/artifacts/buzz-evidence.json" # The relay is host-header tenant-bound (its community row is the authority # of its own RELAY_URL), so agents must present that exact Host. When the # relay actually lives outside the container, this forwarder listens on the @@ -38,6 +49,14 @@ FORWARDER_LOG = f"{REMOTE_LOGS}/relay-forwarder.log" # How many done-poll iterations between in-container liveness probes. LIVENESS_EVERY = 10 +TRANSCRIPT_LIMIT = 1000 +TURN_ENDED_MARKERS = ( + "turn complete for", + "turn cancelled for", + "turn hit max_tokens for", + "turn hit max_turn_requests for", + "turn refused for", +) class RuntimeLaunchError(RuntimeError): @@ -113,14 +132,14 @@ async def run( ) -> RuntimeResult: classes = self._classes_by_agent_id(manifest, trial.credentials) orchestrator = next(c for c in trial.credentials if c.role == "orchestrator") - workers = [c for c in trial.credentials if c.agent_id != orchestrator.agent_id] - if not workers: - raise RuntimeLaunchError("Buzz orchestration requires at least one worker") trial_dir = self.logs_dir / "buzz" trial_dir.mkdir(parents=True, exist_ok=True) agents: list[_Agent] = [] infra: list[_Agent] = [] + task_event_id: str | None = None + final_message: dict[str, Any] | None = None + evidence_exported = False try: await self._install_stack(environment) forwarder = await self._start_forwarder(environment, trial) @@ -163,25 +182,60 @@ async def run( # fail member resolution and kill the trial before the agent # ever saw the task. An explicit --mention demotes unresolved # @-tokens in the text to presentation-only. - await self._send( + task_event = await self._send( trial.user, trial, f"@{orchestrator.agent_id} {instruction}", mention=orchestrator.nostr_pubkey, ) + if isinstance(task_event, dict) and isinstance( + task_event.get("event_id"), str + ): + task_event_id = task_event["event_id"] final_message = await asyncio.wait_for( - self._wait_for_done(environment, orchestrator, trial, agents + infra), + self._wait_for_done( + environment, + orchestrator, + trial, + agents + infra, + solo=agents[0] if len(agents) == 1 else None, + ), timeout=manifest.trial_budget.timeout_seconds, ) await self._verify_m1_output(environment, manifest) finally: await self._stop_agents(environment, agents + infra) await self._collect_logs(environment, trial_dir) + evidence_exported = await self._collect_evidence( + environment=environment, + trial=trial, + trial_dir=trial_dir, + task_event_id=task_event_id, + completion_message_id=( + final_message.get("id") if final_message is not None else None + ), + ) + + # A missing snapshot is a harness failure, not an agent failure: the + # verifier would grade an absent (or agent-planted) artifact as a + # legitimate 0. Fail the trial instead so the two stay distinguishable. + # Scoped to tasks that actually grade the snapshot — a Terminal-Bench + # task is graded by its own tests and must still report its result. + if fixture_for(trial.task_name).requires_evidence and not evidence_exported: + raise RuntimeLaunchError( + "failed to export buzz-evidence.json; the trial has no " + "verifiable relay state and must not be scored" + ) return RuntimeResult( metadata={ - "completion_message_id": final_message["id"], - "completion_message": final_message["content"], + "completion_message_id": ( + final_message.get("id") if final_message is not None else None + ), + "completion_message": ( + final_message.get("content") if final_message is not None else None + ), + "buzz_evidence_exported": evidence_exported, "agent_runtime": "in-container", "agent_hints_enabled": False, "task_seed": "user-identity-prompt", @@ -359,6 +413,7 @@ def _agent_env( """The desktop-launch environment: real acp/agent/dev-mcp wiring.""" return { **endpoint.env, + "RUST_LOG": self._rust_log(endpoint.env.get("RUST_LOG")), "BUZZ_RELAY_URL": trial.relay_ws_url, "BUZZ_PRIVATE_KEY": credential.nostr_secret_key, # Desktop parity: the GUI also sets NOSTR_PRIVATE_KEY on buzz-acp @@ -375,6 +430,9 @@ def _agent_env( "BUZZ_ACP_SYSTEM_PROMPT_FILE": remote_prompt, "BUZZ_AGENT_PROVIDER": endpoint.provider, "BUZZ_AGENT_MODEL": credential.llm_endpoint, + "BUZZ_AGENT_THINKING_EFFORT": ( + agent_class.generation.thinking_effort or THINKING_EFFORT + ), "BUZZ_AGENT_MAX_OUTPUT_TOKENS": str( agent_class.generation.max_output_tokens ), @@ -390,6 +448,15 @@ def _agent_env( endpoint.api_key_env: credential.llm_api_key, } + @staticmethod + def _rust_log(configured: str | None) -> str: + # ``buzz_acp=info`` carries the subscription-readiness line; the turn + # target lets a solo trial stop when its only turn ends. Keep both: + # replacing the former with only the latter makes a healthy process + # look permanently unready. + required = "buzz_acp=info,pool::prompt=info" + return f"{configured},{required}" if configured else required + # -- lifecycle ------------------------------------------------------------- async def _wait_for_agents_ready( @@ -427,11 +494,13 @@ async def _wait_for_done( orchestrator: AgentCredential, trial: TrialHandle, agents: list[_Agent], - ) -> dict[str, Any]: - """Observe the channel as the trial user until the orchestrator posts DONE. + solo: _Agent | None = None, + ) -> dict[str, Any] | None: + """Observe until a team posts DONE or a solo agent finishes its one turn. Observation only: the harness never speaks as any agent. If the team - stalls, the trial times out and the stall is the measured result. + stalls, the trial times out and the stall is the measured result. A solo + agent cannot be woken by a teammate, so its logged turn end is final. """ polls = 0 while True: @@ -453,8 +522,18 @@ async def _wait_for_done( message.get("content", "") ).startswith("DONE:"): return message + if solo is not None and await self._turn_ended(environment, solo): + return None await asyncio.sleep(self.poll_seconds) + @staticmethod + async def _turn_ended(environment: BaseEnvironment, agent: _Agent) -> bool: + result = await environment.exec( + f"cat {shlex.quote(agent.stdout_log)} " + f"{shlex.quote(agent.stderr_log)} 2>/dev/null" + ) + return any(marker in (result.stdout or "") for marker in TURN_ENDED_MARKERS) + async def _raise_for_dead_agents( self, environment: BaseEnvironment, agents: list[_Agent] ) -> None: @@ -503,6 +582,120 @@ async def _collect_logs( except Exception: # noqa: S110, BLE001 — best effort; env may be torn down pass + async def _collect_evidence( + self, + *, + environment: BaseEnvironment, + trial: TrialHandle, + trial_dir: Path, + task_event_id: str | None, + completion_message_id: str | None, + ) -> bool: + """Snapshot public relay state for the verifier before trial teardown.""" + try: + messages = await self._buzz_json( + trial.user, + trial, + "messages", + "get", + "--channel", + trial.channel_id, + "--limit", + str(TRANSCRIPT_LIMIT), + ) + observed_channels = await self._collect_observed_channels(trial) + evidence = build_buzz_evidence( + trial=trial, + messages=messages, + task_event_id=task_event_id, + completion_message_id=completion_message_id, + transcript_limit=TRANSCRIPT_LIMIT, + observed_channels=observed_channels, + ) + evidence_path = trial_dir / "buzz-evidence.json" + evidence_path.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + transcript = { + "channel_id": trial.channel_id, + "message_count": evidence["message_count"], + "truncated": evidence["truncated"], + "messages": evidence["messages"], + } + (trial_dir / "transcript.json").write_text( + json.dumps(transcript, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + result = await environment.exec("mkdir -p /logs/artifacts") + if result.return_code != 0: + self._record_evidence_error( + trial_dir, f"mkdir /logs/artifacts exited {result.return_code}" + ) + return False + await environment.upload_file(evidence_path, REMOTE_EVIDENCE) + return True + except Exception: # noqa: BLE001 — the caller fails the trial + self._record_evidence_error(trial_dir, traceback.format_exc()) + return False + + @staticmethod + def _record_evidence_error(trial_dir: Path, reason: str) -> None: + """Persist why the snapshot failed; the caller only sees a bool.""" + try: + (trial_dir / "buzz-evidence-error.txt").write_text(reason, encoding="utf-8") + except OSError: + # Diagnostics only — never mask the failure we are reporting. + pass + + async def _collect_observed_channels( + self, trial: TrialHandle + ) -> list[dict[str, Any]]: + """Read task-declared channel state through the production CLI.""" + names = fixture_for(trial.task_name).observe_channel_names + if not names: + return [] + orchestrator = next( + credential + for credential in trial.credentials + if credential.role == "orchestrator" + ) + observed: list[dict[str, Any]] = [] + for name in names: + matches = await self._buzz_json( + orchestrator, + trial, + "channels", + "search", + "--query", + name, + "--exact", + "--include-archived", + ) + if not isinstance(matches, list): + continue + for match in matches: + if not isinstance(match, dict): + continue + channel_id = match.get("channel_id") + if not isinstance(channel_id, str) or not channel_id: + continue + members = await self._buzz_json( + orchestrator, + trial, + "channels", + "members", + "--channel", + channel_id, + ) + observed.append( + { + **match, + "members": members if isinstance(members, list) else [], + } + ) + return observed + # -- Buzz CLI as the trial user / provisioning identities ------------------- @staticmethod @@ -533,7 +726,7 @@ async def _send( content: str, *, mention: str | None = None, - ) -> None: + ) -> Any: args = [ "messages", "send", @@ -544,7 +737,7 @@ async def _send( ] if mention is not None: args += ["--mention", mention] - await self._buzz_json(credential, trial, *args) + return await self._buzz_json(credential, trial, *args) async def _buzz_json( self, credential: AgentCredential, trial: TrialHandle, *args: str diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/evidence.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/evidence.py new file mode 100644 index 00000000000..45eaaa3cfa9 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/evidence.py @@ -0,0 +1,130 @@ +"""Stable, verifier-facing evidence derived from a Buzz channel transcript.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +from .provisioning import TrialHandle + +EVIDENCE_SCHEMA_VERSION = 1 + + +def _tags(message: Mapping[str, Any]) -> list[list[str]]: + """Return only well-formed string tags from a relay message.""" + raw = message.get("tags") + if not isinstance(raw, list): + return [] + return [ + list(tag) + for tag in raw + if isinstance(tag, list) + and tag + and all(isinstance(value, str) for value in tag) + ] + + +def _tag_value( + tags: Iterable[list[str]], name: str, marker: str | None = None +) -> str | None: + for tag in tags: + if len(tag) < 2 or tag[0] != name: + continue + if marker is not None and (len(tag) < 4 or tag[3] != marker): + continue + return tag[1] + return None + + +def _normalize_message( + message: Mapping[str, Any], identities: Mapping[str, Mapping[str, str]] +) -> dict[str, Any]: + tags = _tags(message) + pubkey = message.get("pubkey") if isinstance(message.get("pubkey"), str) else "" + identity = identities.get(pubkey, {}) + return { + "id": message.get("id") if isinstance(message.get("id"), str) else "", + "kind": message.get("kind") if isinstance(message.get("kind"), int) else None, + "created_at": ( + message.get("created_at") + if isinstance(message.get("created_at"), int) + else None + ), + "pubkey": pubkey, + "author": identity.get("name", "unknown"), + "author_role": identity.get("role", "unknown"), + "content": ( + message.get("content") if isinstance(message.get("content"), str) else "" + ), + # Preserve the signed protocol evidence. Derived fields below make the + # common checks convenient without replacing the source-of-truth tags. + "tags": tags, + "channel_id": _tag_value(tags, "h"), + "reply_to_event_id": _tag_value(tags, "e", "reply"), + "mentioned_pubkeys": [ + tag[1] for tag in tags if len(tag) >= 2 and tag[0] == "p" + ], + } + + +def build_buzz_evidence( + *, + trial: TrialHandle, + messages: object, + task_event_id: str | None, + completion_message_id: str | None, + transcript_limit: int, + observed_channels: object = None, +) -> dict[str, Any]: + """Normalize relay messages into a versioned contract for task verifiers. + + Private keys and auth tags are intentionally absent. The exported identities + contain only public names, roles, and pubkeys already visible on the relay. + """ + raw_messages = messages if isinstance(messages, list) else [] + identity_rows = ( + (trial.user.agent_id, "user", trial.user.nostr_pubkey), + *( + (credential.agent_id, credential.role, credential.nostr_pubkey) + for credential in trial.credentials + ), + ) + identities_by_pubkey = { + pubkey: {"name": name, "role": role} for name, role, pubkey in identity_rows + } + identities = { + name: {"role": role, "pubkey": pubkey} for name, role, pubkey in identity_rows + } + normalized = [ + _normalize_message(message, identities_by_pubkey) + for message in raw_messages + if isinstance(message, dict) + ] + normalized.sort( + key=lambda message: ( + message["created_at"] is None, + message["created_at"] or 0, + ) + ) + return { + "schema_version": EVIDENCE_SCHEMA_VERSION, + "trial": { + "run_id": trial.run_id, + "trial_id": trial.trial_id, + "channel_id": trial.channel_id, + }, + "task_event_id": task_event_id, + "completion_message_id": completion_message_id, + "identities": identities, + "directory": [ + {"name": identity.name, "role": identity.role, "pubkey": identity.pubkey} + for identity in trial.directory + ], + "task_name": trial.task_name, + "observed_channels": ( + observed_channels if isinstance(observed_channels, list) else [] + ), + "message_count": len(normalized), + "truncated": len(raw_messages) >= transcript_limit, + "messages": normalized, + } diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py index 309c0a5770c..fc0d40de4fd 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py @@ -34,6 +34,15 @@ class GenerationConfig(StrictModel): temperature: float = Field(default=0.0, ge=0.0) max_output_tokens: int = Field(gt=0) context_window_tokens: int = Field(gt=0) + # Reasoning effort pinned per condition. buzz-agent clamps an unsupported + # level to the nearest one the model accepts and only warns, so a condition + # asking for more than the endpoint supports runs silently at less. + # Unset means the runtime's default, which is pinned rather than left to + # the provider: a provider default is neither recorded nor stable across + # endpoints. + thinking_effort: ( + Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] | None + ) = None extra: dict[str, Any] = Field(default_factory=dict) @@ -113,6 +122,18 @@ def validate_roster(self) -> Self: def canonical_bytes(self) -> bytes: """Return stable UTF-8 JSON independent of YAML formatting and key order.""" data = self.model_dump(mode="json", exclude_none=False) + # An unpinned `thinking_effort` is dropped rather than serialised as + # null: the hash answers "are these two runs the same experiment?", and + # a manifest written before this field existed sends a byte-identical + # container environment, so opening the effort axis must not + # re-identify every condition that does not use it. + for entry in data.get("roster", []): + generation = entry.get("generation") + if ( + isinstance(generation, dict) + and generation.get("thinking_effort") is None + ): + generation.pop("thinking_effort", None) return json.dumps( data, sort_keys=True, diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/provisioning.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/provisioning.py index 37e9bbe4780..b27ae8ae122 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/provisioning.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/provisioning.py @@ -21,6 +21,15 @@ class AgentCredential: llm_api_key: str +@dataclass(frozen=True, slots=True) +class DirectoryIdentity: + """One public, benchmark-seeded identity discoverable through Buzz.""" + + name: str + role: str + pubkey: str + + @dataclass(frozen=True, slots=True) class TrialHandle: """Provisioned Buzz resources owned by one Harbor trial.""" @@ -39,6 +48,9 @@ class TrialHandle: # identity and the harness run. ``relay_ws_url`` is the view from the # agents' runtime (the task container). Empty means both views coincide. user_relay_url: str = "" + # Additive Buzz-native task context. Directory entries contain no secrets. + task_name: str = "" + directory: tuple[DirectoryIdentity, ...] = () @runtime_checkable @@ -51,6 +63,7 @@ def create_trial( trial_id: str, manifest: ExperimentManifest, channel_label: str | None = None, + task_name: str | None = None, ) -> TrialHandle: ... def teardown(self, handle: TrialHandle) -> None: ... diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py new file mode 100644 index 00000000000..afcfb6f050d --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py @@ -0,0 +1,68 @@ +"""Public setup declarations for Buzz-native benchmark tasks.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class DirectoryEntry: + """A named identity to seed into the benchmark community.""" + + name: str + role: str + + +@dataclass(frozen=True, slots=True) +class BuzzTaskFixture: + """Relay state a task needs before the agent receives its prompt.""" + + directory: tuple[DirectoryEntry, ...] = () + observe_channel_names: tuple[str, ...] = () + user_display_name: str | None = None + # Whether the task's verifier grades the exported relay snapshot. Only + # these tasks fail when the export fails; a Terminal-Bench task is graded + # by its own tests and must not be errored by a snapshot hiccup. + requires_evidence: bool = False + + +CREATE_CHANNEL_TASK = "create-channel-invite-users" +CREATE_CHANNEL_NAME = "fix-pr-1234" +TARGET_USERS = ("benchmark-user-07", "benchmark-user-19", "benchmark-user-42") +TARGET_BOTS = ("benchmark-bot-03", "benchmark-bot-08") +USER_MENTION_TASK = "user-mention" +USER_MENTION_DISPLAY_NAME = "John Vincent Doe" +REPLY_TO_THREAD_TASK = "reply-to-thread" +READ_NAMED_PATH_TASK = "read-named-path-outside-workspace" + +_CREATE_CHANNEL_FIXTURE = BuzzTaskFixture( + directory=tuple( + [ + DirectoryEntry(f"benchmark-user-{index:02d}", "user") + for index in range(1, 51) + ] + + [ + DirectoryEntry(f"benchmark-bot-{index:02d}", "bot") + for index in range(1, 11) + ] + ), + observe_channel_names=(CREATE_CHANNEL_NAME,), + requires_evidence=True, +) + +_USER_MENTION_FIXTURE = BuzzTaskFixture( + user_display_name=USER_MENTION_DISPLAY_NAME, + requires_evidence=True, +) + +_FIXTURES = { + CREATE_CHANNEL_TASK: _CREATE_CHANNEL_FIXTURE, + USER_MENTION_TASK: _USER_MENTION_FIXTURE, + REPLY_TO_THREAD_TASK: BuzzTaskFixture(requires_evidence=True), + READ_NAMED_PATH_TASK: BuzzTaskFixture(requires_evidence=True), +} + + +def fixture_for(task_name: str | None) -> BuzzTaskFixture: + """Return the declared setup for a task, or an empty setup.""" + return _FIXTURES.get(task_name or "", BuzzTaskFixture()) diff --git a/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json new file mode 100644 index 00000000000..05fc0dc2624 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live.json @@ -0,0 +1,7 @@ +{ + "gpt-5.6-luna": { + "provider": "openai", + "api_key_env": "OPENAI_COMPAT_API_KEY", + "env": {} + } +} diff --git a/benchmarks/harbor-buzz-orchestra/testbed/pyproject.toml b/benchmarks/harbor-buzz-orchestra/testbed/pyproject.toml index 934e7845ff0..1fe56021e90 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/pyproject.toml +++ b/benchmarks/harbor-buzz-orchestra/testbed/pyproject.toml @@ -16,7 +16,7 @@ build-backend = "hatchling.build" [project.optional-dependencies] dev = [ "pytest>=8.4", - "ruff>=0.15", + "ruff==0.16.3", ] [tool.uv.sources] diff --git a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py index bd11f193cae..f1c4077d7e2 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py @@ -93,6 +93,17 @@ def add_member(self, channel_id: str, pubkey: str) -> None: "member", ) + def profiles(self, pubkeys: list[str]) -> list[dict[str, Any]]: + """Return the profiles currently published for the given pubkeys.""" + args = ["users", "get"] + for pubkey in pubkeys: + args.extend(("--pubkey", pubkey)) + response = self.run(*args) + return response if isinstance(response, list) else [] + + def set_profile(self, name: str) -> None: + self.run("users", "set-profile", "--name", name) + def archive_channel(self, channel_id: str) -> None: self.run("channels", "archive", "--channel", channel_id) diff --git a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py index d8f380387d3..11d0c886190 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py @@ -11,7 +11,12 @@ import psycopg from harbor_buzz_orchestra.manifest import ExperimentManifest -from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle +from harbor_buzz_orchestra.provisioning import ( + AgentCredential, + DirectoryIdentity, + TrialHandle, +) +from harbor_buzz_orchestra.task_fixtures import fixture_for from .buzz_cli import BuzzCli from .keys import compute_auth_tag, generate_keypair, keypair_from_secret @@ -73,6 +78,7 @@ def create_trial( trial_id: str, manifest: ExperimentManifest, channel_label: str | None = None, + task_name: str | None = None, ) -> TrialHandle: manifest_hash = manifest.sha256 with psycopg.connect(self._config.postgres_dsn) as conn: @@ -87,7 +93,12 @@ def create_trial( return existing handle = self._provision( - run_id, trial_id, manifest, manifest_hash, channel_label + run_id, + trial_id, + manifest, + manifest_hash, + channel_label, + task_name, ) self._store_trial(conn, handle) conn.commit() @@ -139,9 +150,10 @@ def _provision( manifest: ExperimentManifest, manifest_hash: str, channel_label: str | None, + task_name: str | None, ) -> TrialHandle: credentials = self._mint_credentials(manifest) - user = self._mint_user() + user = self._mint_user(task_name) # The user identity creates the channel and invites the agents — # mirroring production Buzz, where a human owns the channel their # agents work in. @@ -157,6 +169,7 @@ def _provision( ) for credential in credentials: cli.add_member(channel_id, credential.nostr_pubkey) + directory = self._seed_directory(task_name, cli) return TrialHandle( run_id=run_id, trial_id=trial_id, @@ -166,6 +179,66 @@ def _provision( credentials=credentials, user=user, user_relay_url=self._config.relay_http_url, + task_name=task_name or "", + directory=directory, + ) + + def _seed_directory( + self, task_name: str | None, observer: BuzzCli + ) -> tuple[DirectoryIdentity, ...]: + """Publish stable task-directory profiles, skipping those already seeded.""" + entries = fixture_for(task_name).directory + credentials = [ + self._directory_credential(entry.name, entry.role) for entry in entries + ] + if not credentials: + return () + existing = { + profile.get("pubkey") + for profile in observer.profiles( + [credential.nostr_pubkey for credential in credentials] + ) + if isinstance(profile, dict) + } + for credential in credentials: + if credential.nostr_pubkey not in existing: + self._cli_for(credential).set_profile(credential.agent_id) + return tuple( + DirectoryIdentity( + name=credential.agent_id, + role=credential.role, + pubkey=credential.nostr_pubkey, + ) + for credential in credentials + ) + + def _directory_credential(self, name: str, role: str) -> AgentCredential: + """Derive one community-stable benchmark identity without storing its key.""" + return self._stable_credential(name, name, role) + + def _stable_credential( + self, identity_id: str, display_name: str, role: str + ) -> AgentCredential: + """Derive an owner-scoped stable identity for reusable task fixtures.""" + order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 + digest = hashlib.sha256( + b"buzz-benchmark-directory-v1\0" + + bytes.fromhex(self._config.owner_secret_key) + + b"\0" + + identity_id.encode() + ).digest() + secret = ((int.from_bytes(digest, "big") % (order - 1)) + 1).to_bytes(32, "big") + keypair = keypair_from_secret(secret.hex()) + return AgentCredential( + agent_id=display_name, + role=role, + nostr_secret_key=keypair.secret_key, + nostr_pubkey=keypair.pubkey, + nostr_auth_tag=compute_auth_tag( + self._config.owner_secret_key, keypair.pubkey + ), + llm_endpoint="", + llm_api_key="", ) def _mint_credentials( @@ -196,13 +269,21 @@ def _mint_credentials( ) return tuple(credentials) - def _mint_user(self) -> AgentCredential: + def _mint_user(self, task_name: str | None = None) -> AgentCredential: """Mint the trial's user identity — the human analogue, not an agent. + A task may declare a dedicated stable identity when its user-facing + profile is part of what the benchmark measures. This avoids profile + races with the pinned GUI user when different tasks run concurrently. With a pinned ``user_secret_key`` the same identity fronts every trial, like one human running many teams; otherwise each trial gets a fresh user key. """ + display_name = fixture_for(task_name).user_display_name + if display_name is not None: + return self._stable_credential( + f"task-user:{task_name}", display_name, "user" + ) keypair = ( keypair_from_secret(self._config.user_secret_key) if self._config.user_secret_key @@ -256,6 +337,11 @@ def _load_trial( ), user=AgentCredential(**stored["user"]), user_relay_url=stored.get("user_relay_url", ""), + task_name=stored.get("task_name", ""), + directory=tuple( + DirectoryIdentity(**identity) + for identity in stored.get("directory", []) + ), ) @staticmethod diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py index e784de58256..1f55526cf3c 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py @@ -68,6 +68,19 @@ def test_mint_user_is_attested_and_not_an_agent(): assert tag[:3] == ["auth", owner_pubkey.format().hex(), ""] +def test_user_mention_task_gets_stable_three_word_user_identity(): + provisioner = BuzzTrialProvisioner(config(user_secret_key="7" * 64)) + + first = provisioner._mint_user("user-mention") + second = provisioner._mint_user("user-mention") + + assert first.agent_id == "John Vincent Doe" + assert len(first.agent_id.split()) == 3 + assert first.nostr_secret_key == second.nostr_secret_key + assert first.nostr_secret_key != "7" * 64 + assert first.role == "user" + + def test_pinned_user_secret_reuses_one_identity(): pinned = "7" * 64 provisioner = BuzzTrialProvisioner(config(user_secret_key=pinned)) @@ -96,6 +109,44 @@ def test_mint_credentials_missing_api_key_is_explicit(manifest): provisioner._mint_credentials(manifest) +def test_directory_credentials_are_stable_distinct_and_attested(): + provisioner = BuzzTrialProvisioner(config()) + + first = provisioner._directory_credential("benchmark-user-01", "user") + again = provisioner._directory_credential("benchmark-user-01", "user") + other = provisioner._directory_credential("benchmark-bot-01", "bot") + + assert first.nostr_secret_key == again.nostr_secret_key + assert first.nostr_pubkey == again.nostr_pubkey + assert first.nostr_pubkey != other.nostr_pubkey + assert first.role == "user" and other.role == "bot" + assert json.loads(first.nostr_auth_tag)[2] == "" + + +def test_seed_directory_has_50_users_10_bots_and_skips_existing(monkeypatch): + provisioner = BuzzTrialProvisioner(config()) + + class Observer: + def profiles(self, pubkeys): + return [{"pubkey": pubkeys[0]}] + + published = [] + + class Publisher: + def set_profile(self, name): + published.append(name) + + monkeypatch.setattr(provisioner, "_cli_for", lambda _credential: Publisher()) + + directory = provisioner._seed_directory("create-channel-invite-users", Observer()) + + assert len(directory) == 60 + assert sum(identity.role == "user" for identity in directory) == 50 + assert sum(identity.role == "bot" for identity in directory) == 10 + assert len(published) == 59 + assert directory[0].name not in published + + def test_lock_key_is_deterministic_and_distinct(): calls: list[int] = [] diff --git a/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/threaded.json b/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/threaded.json new file mode 100644 index 00000000000..9f7d045ddd5 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/threaded.json @@ -0,0 +1,32 @@ +{ + "channel_id": "6a178caa-07de-4594-9296-1f130b3f32e2", + "message_count": 2, + "truncated": false, + "messages": [ + { + "id": "585eeddbd9c1384696a615faeafdca5c00ff806276c8c6c66b654e7bbb40e167", + "author": "user", + "pubkey": "779c3f730c638f67cc06c7e7d55d31720c85c9eb74cec2050d0ac5fdabaafea8", + "content": "@solo-1 Complete the requested task.", + "created_at": 1786995079, + "kind": 9, + "tags": [ + ["h", "6a178caa-07de-4594-9296-1f130b3f32e2"], + ["p", "ed8ce3ee42988114b5940d9dd0023d576649e06c47bc018ef74933b2472e4854"] + ] + }, + { + "id": "220c6ac96cf28a995cc14ff8b72d5ec7d968c0a859eeaa0d2ec354670def60e5", + "author": "solo-1", + "pubkey": "ed8ce3ee42988114b5940d9dd0023d576649e06c47bc018ef74933b2472e4854", + "content": "DONE: Completed the requested task.", + "created_at": 1786995113, + "kind": 9, + "tags": [ + ["h", "6a178caa-07de-4594-9296-1f130b3f32e2"], + ["e", "585eeddbd9c1384696a615faeafdca5c00ff806276c8c6c66b654e7bbb40e167", "", "reply"], + ["p", "779c3f730c638f67cc06c7e7d55d31720c85c9eb74cec2050d0ac5fdabaafea8"] + ] + } + ] +} diff --git a/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/top-level.json b/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/top-level.json new file mode 100644 index 00000000000..a00d5da78b2 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/fixtures/transcripts/top-level.json @@ -0,0 +1,30 @@ +{ + "channel_id": "91879837-0cae-4c42-ab42-b01fe6ff8f35", + "message_count": 2, + "truncated": false, + "messages": [ + { + "id": "774c8ea010c6cdf4b6cbd667463d5a62371de00ac8913af190b8988b48f2b230", + "author": "user", + "pubkey": "d56c715eb650f6e880851b3f37e9a742f29436a085e925d80c6d0904a397fb0f", + "content": "@solo-1 Complete the requested task.", + "created_at": 1786993144, + "kind": 9, + "tags": [ + ["h", "91879837-0cae-4c42-ab42-b01fe6ff8f35"], + ["p", "40ed137e6e725207a63905cbbdbfe99497c28aa16eec8efbeeff49145ad575c8"] + ] + }, + { + "id": "943fefebec06504c99a48f704e77fe246cd81a0e90f5e3d6a1d073554a812c0d", + "author": "solo-1", + "pubkey": "40ed137e6e725207a63905cbbdbfe99497c28aa16eec8efbeeff49145ad575c8", + "content": "DONE: Completed the requested task.", + "created_at": 1786993179, + "kind": 9, + "tags": [ + ["h", "91879837-0cae-4c42-ab42-b01fe6ff8f35"] + ] + } + ] +} diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py index b305344c51c..184cc0194eb 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py @@ -37,8 +37,10 @@ def __init__(self): def healthcheck(self): self.healthchecked = True - def create_trial(self, run_id, trial_id, manifest, channel_label=None): - self.created = (run_id, trial_id, manifest, channel_label) + def create_trial( + self, run_id, trial_id, manifest, channel_label=None, task_name=None + ): + self.created = (run_id, trial_id, manifest, channel_label, task_name) return TrialHandle( run_id, trial_id, @@ -91,6 +93,7 @@ async def test_agent_lifecycle_and_context(tmp_path, manifest_data): assert provisioner.created[:2] == ("run-1", str(context_id)) # The task short name labels the trial channel for spectator GUIs. assert provisioner.created[3] == "hello-world" + assert provisioner.created[4] == "hello-world" assert provisioner.torn_down.channel_id == "channel-1" assert runtime.called["instruction"] == "solve it" assert ( diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index c0f5beeef22..4f91c974af6 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -3,6 +3,7 @@ import hashlib import json import re +from dataclasses import replace from pathlib import Path import pytest @@ -10,13 +11,16 @@ from harbor_buzz_orchestra.container_runtime import ( REMOTE_BIN, + REMOTE_EVIDENCE, REMOTE_LOGS, + THINKING_EFFORT, BuzzContainerRuntime, EndpointLaunchConfig, RuntimeLaunchError, ) from harbor_buzz_orchestra.manifest import ExperimentManifest from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle +from harbor_buzz_orchestra.task_fixtures import fixture_for def write_manifest(tmp_path: Path) -> ExperimentManifest: @@ -173,6 +177,51 @@ def test_user_relay_url_prefers_host_view(tmp_path): ) # pre-v1.2 handles fall back to deriving http from the agents' ws view. assert rt._user_relay_url(trial_handle(())) == "http://host.docker.internal:3600" + + +async def test_collects_task_declared_channel_membership(tmp_path, monkeypatch): + rt = runtime(tmp_path) + trial = replace( + trial_handle((credential("orch-1", "orchestrator", "orch-model"),)), + task_name="create-channel-invite-users", + ) + calls = [] + + async def buzz_json(credential_arg, trial_arg, *args): + calls.append((credential_arg, trial_arg, args)) + if args[:2] == ("channels", "search"): + return [ + { + "channel_id": "created-channel", + "name": "fix-pr-1234", + "channel_type": "stream", + "visibility": "private", + "archived": False, + "ttl_seconds": 3600, + } + ] + return [{"pubkey": "member", "role": "member"}] + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + + observed = await rt._collect_observed_channels(trial) + + assert observed[0]["members"] == [{"pubkey": "member", "role": "member"}] + assert calls[0][0].agent_id == "orch-1" + assert calls[0][2] == ( + "channels", + "search", + "--query", + "fix-pr-1234", + "--exact", + "--include-archived", + ) + assert calls[1][2] == ( + "channels", + "members", + "--channel", + "created-channel", + ) with pytest.raises(RuntimeLaunchError, match="ws://"): rt._cli_relay_url("http://relay") @@ -375,9 +424,7 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped( assert bool(probed) == (condition == "M1-hello-world") -async def test_send_mentions_by_pubkey_so_task_text_stays_inert( - tmp_path, monkeypatch -): +async def test_send_mentions_by_pubkey_so_task_text_stays_inert(tmp_path, monkeypatch): """Task text is untrusted payload: `:%normal! @a` in a task statement must not be fed to member-name resolution (it would fail and kill the trial). An explicit --mention pins delivery to the orchestrator's pubkey.""" @@ -429,6 +476,109 @@ async def buzz_json(credential, *args, **kwargs): assert set(observers) == {"user"} +async def test_solo_turn_end_completes_without_done_message(tmp_path, monkeypatch): + from harbor_buzz_orchestra.container_runtime import _Agent + + rt = runtime(tmp_path, poll_seconds=0) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + solo = _Agent(orch, 7, "stdout.log", "stderr.log") + environment = Environment( + responses={ + "cat ": ExecResult( + stdout="turn complete for channel: end_turn\n", + stderr="", + return_code=0, + ) + } + ) + + async def buzz_json(*args, **kwargs): + return [] + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + assert await rt._wait_for_done(environment, orch, trial, [], solo=solo) is None + + +async def test_collect_evidence_uploads_verifier_artifact(tmp_path, monkeypatch): + rt = runtime(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + root_id = "root-event" + reply_id = "reply-event" + messages = [ + { + "id": root_id, + "kind": 9, + "created_at": 1, + "pubkey": trial.user.nostr_pubkey, + "content": "question", + "tags": [["h", trial.channel_id], ["p", orch.nostr_pubkey]], + }, + { + "id": reply_id, + "kind": 9, + "created_at": 2, + "pubkey": orch.nostr_pubkey, + "content": "answer", + "tags": [["h", trial.channel_id], ["e", root_id, "", "reply"]], + }, + ] + + async def buzz_json(*args, **kwargs): + return messages + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + environment = Environment() + trial_dir = tmp_path / "trial" + trial_dir.mkdir() + + assert await rt._collect_evidence( + environment=environment, + trial=trial, + trial_dir=trial_dir, + task_event_id=root_id, + completion_message_id=reply_id, + ) + assert environment.uploads[-1][1] == REMOTE_EVIDENCE + evidence = json.loads((trial_dir / "buzz-evidence.json").read_text()) + assert evidence["messages"][-1]["reply_to_event_id"] == root_id + assert (trial_dir / "transcript.json").is_file() + + +async def test_failed_evidence_snapshot_records_the_reason(tmp_path, monkeypatch): + rt = runtime(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + + async def buzz_json(*args, **kwargs): + raise RuntimeError("relay unreachable") + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + trial_dir = tmp_path / "trial" + trial_dir.mkdir() + + assert not await rt._collect_evidence( + environment=Environment(), + trial=trial, + trial_dir=trial_dir, + task_event_id="root-event", + completion_message_id=None, + ) + # The caller only sees a bool, so the cause has to survive as an artifact — + # otherwise a failed export is indistinguishable from a quiet relay. + assert "relay unreachable" in (trial_dir / "buzz-evidence-error.txt").read_text() + assert not (trial_dir / "buzz-evidence.json").exists() + + +def test_runtime_logging_keeps_readiness_and_turn_completion_signals(tmp_path): + rt = runtime(tmp_path) + assert rt._rust_log(None) == "buzz_acp=info,pool::prompt=info" + assert rt._rust_log("custom=debug") == ( + "custom=debug,buzz_acp=info,pool::prompt=info" + ) + + def test_composed_system_prompt_carries_persona_and_team_roster(tmp_path): rt = runtime(tmp_path) orch = credential("orch-1", "orchestrator", "orch-model") @@ -466,3 +616,43 @@ async def test_stop_agents_sweeps_the_uploaded_stack(tmp_path): sweeps = [cmd for cmd, _ in environment.commands if REMOTE_BIN in cmd] assert len(sweeps) == 2 assert "kill -TERM" in sweeps[0] and "kill -KILL" in sweeps[1] + + +def test_only_evidence_grading_tasks_fail_on_a_missing_snapshot(): + # Terminal-Bench tasks share this runtime but are graded by their own + # tests, so a snapshot hiccup must not turn a real result into an error. + assert fixture_for("reply-to-thread").requires_evidence + assert fixture_for("user-mention").requires_evidence + assert fixture_for("read-named-path-outside-workspace").requires_evidence + assert fixture_for("create-channel-invite-users").requires_evidence + assert not fixture_for("cobol-modernization").requires_evidence + assert not fixture_for(None).requires_evidence + + +@pytest.mark.parametrize( + ("pinned", "expected"), [(None, THINKING_EFFORT), ("high", "high")] +) +async def test_thinking_effort_reaches_the_agent(tmp_path, pinned, expected): + manifest = write_manifest(tmp_path) + agent_class = manifest.roster[0] + if pinned is not None: + agent_class = agent_class.model_copy( + update={ + "generation": agent_class.generation.model_copy( + update={"thinking_effort": pinned} + ) + } + ) + orch = credential("orch-1", "orchestrator", "orch-model") + environment = Environment( + responses={"buzz-acp": ExecResult(stdout="4242\n", stderr="", return_code=0)} + ) + await runtime(tmp_path)._launch_agent( + environment=environment, + trial=trial_handle((orch,)), + credential=orch, + agent_class=agent_class, + trial_dir=tmp_path, + ) + _, env = environment.commands[-1] + assert env["BUZZ_AGENT_THINKING_EFFORT"] == expected diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_create_channel_invite_users_verifier.py b/benchmarks/harbor-buzz-orchestra/tests/test_create_channel_invite_users_verifier.py new file mode 100644 index 00000000000..7910ab221b2 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/test_create_channel_invite_users_verifier.py @@ -0,0 +1,117 @@ +import copy +import importlib.util +from pathlib import Path + +from harbor_buzz_orchestra.task_fixtures import TARGET_BOTS, TARGET_USERS, fixture_for + +# The Buzz-native tasks are a sibling dataset of this harness package. +DATASET_ROOT = Path(__file__).resolve().parents[2] / "buzz-dataset" +VERIFIER = DATASET_ROOT / "create-channel-invite-users" / "tests" / "verify.py" +SPEC = importlib.util.spec_from_file_location("create_channel_verifier", VERIFIER) +verifier = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(verifier) + +ORCHESTRATOR = "a" * 64 + + +def _evidence() -> dict: + fixture = fixture_for("create-channel-invite-users") + directory = [ + {"name": entry.name, "role": entry.role, "pubkey": f"{index:064x}"} + for index, entry in enumerate(fixture.directory, start=1) + ] + by_name = {entry["name"]: entry for entry in directory} + members = [{"pubkey": ORCHESTRATOR, "role": "owner"}] + members += [ + {"pubkey": by_name[name]["pubkey"], "role": "member"} for name in TARGET_USERS + ] + members += [ + {"pubkey": by_name[name]["pubkey"], "role": "bot"} for name in TARGET_BOTS + ] + return { + "schema_version": 1, + "task_name": "create-channel-invite-users", + "identities": {"solo-1": {"role": "orchestrator", "pubkey": ORCHESTRATOR}}, + "directory": directory, + "observed_channels": [ + { + "channel_id": "channel-1", + "name": "fix-pr-1234", + "channel_type": "stream", + "visibility": "private", + "archived": False, + "ttl_seconds": 3600, + "members": members, + } + ], + } + + +def test_exact_temporary_channel_and_roster_passes(): + metrics, details = verifier.score_evidence(_evidence()) + + assert all(value == 1.0 for value in metrics.values()) + assert details["channel_id"] == "channel-1" + + +def test_extra_member_fails_exact_membership(): + evidence = _evidence() + evidence["observed_channels"][0]["members"].append( + {"pubkey": "f" * 64, "role": "member"} + ) + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["channel_created"] == 1.0 + assert metrics["exact_membership"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_wrong_bot_role_fails_roles(): + evidence = _evidence() + bot_pubkey = next( + row["pubkey"] for row in evidence["directory"] if row["name"] == TARGET_BOTS[0] + ) + member = next( + row + for row in evidence["observed_channels"][0]["members"] + if row["pubkey"] == bot_pubkey + ) + member["role"] = "member" + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["exact_membership"] == 1.0 + assert metrics["expected_roles"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_permanent_channel_fails_temporary_requirement(): + evidence = _evidence() + evidence["observed_channels"][0]["ttl_seconds"] = None + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["temporary_channel"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_duplicate_exact_name_fails_channel_creation(): + evidence = _evidence() + evidence["observed_channels"].append( + copy.deepcopy(evidence["observed_channels"][0]) + ) + + metrics, details = verifier.score_evidence(evidence) + + assert details["matching_channel_count"] == 2 + assert metrics["channel_created"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_missing_evidence_fails_closed(): + metrics, details = verifier.score_evidence(None) + + assert all(value == 0.0 for value in metrics.values()) + assert "error" in details diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_evidence.py b/benchmarks/harbor-buzz-orchestra/tests/test_evidence.py new file mode 100644 index 00000000000..3e67f43a3bc --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/test_evidence.py @@ -0,0 +1,129 @@ +import json +from pathlib import Path + +from harbor_buzz_orchestra.evidence import build_buzz_evidence +from harbor_buzz_orchestra.provisioning import ( + AgentCredential, + DirectoryIdentity, + TrialHandle, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "transcripts" + + +def _credential(agent_id: str, role: str, pubkey: str) -> AgentCredential: + return AgentCredential( + agent_id=agent_id, + role=role, + nostr_secret_key=f"secret-{agent_id}", + nostr_pubkey=pubkey, + nostr_auth_tag=f"auth-{agent_id}", + llm_endpoint="model" if role != "user" else "", + llm_api_key="key" if role != "user" else "", + ) + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def _trial(transcript: dict) -> TrialHandle: + user_message, agent_message = transcript["messages"] + return TrialHandle( + run_id="run-1", + trial_id="trial-1", + manifest_hash="hash", + relay_ws_url="ws://relay", + channel_id=transcript["channel_id"], + credentials=(_credential("solo-1", "orchestrator", agent_message["pubkey"]),), + user=_credential("user", "user", user_message["pubkey"]), + ) + + +def _evidence(name: str) -> dict: + transcript = _load(name) + return build_buzz_evidence( + trial=_trial(transcript), + messages=list(reversed(transcript["messages"])), + task_event_id=transcript["messages"][0]["id"], + completion_message_id=transcript["messages"][-1]["id"], + transcript_limit=1000, + ) + + +def test_normalizes_real_threaded_transcript_and_preserves_protocol_tags(): + evidence = _evidence("threaded.json") + + assert evidence["schema_version"] == 1 + assert evidence["message_count"] == 2 + assert evidence["messages"][0]["author_role"] == "user" + reply = evidence["messages"][-1] + assert reply["author"] == "solo-1" + assert reply["author_role"] == "orchestrator" + assert reply["channel_id"] == evidence["trial"]["channel_id"] + assert reply["reply_to_event_id"] == evidence["task_event_id"] + assert ["e", evidence["task_event_id"], "", "reply"] in reply["tags"] + + +def test_top_level_agent_message_has_no_derived_reply_destination(): + evidence = _evidence("top-level.json") + + assert evidence["messages"][-1]["reply_to_event_id"] is None + + +def test_malformed_messages_are_safe_and_secrets_are_never_exported(): + transcript = _load("threaded.json") + trial = _trial(transcript) + evidence = build_buzz_evidence( + trial=trial, + messages=[None, {"id": "broken", "tags": ["bad", ["e", 7]]}], + task_event_id=None, + completion_message_id=None, + transcript_limit=1, + ) + + assert evidence["message_count"] == 1 + assert evidence["messages"][0]["tags"] == [] + assert evidence["truncated"] is True + encoded = json.dumps(evidence) + assert "nostr_secret_key" not in encoded + assert "auth-user" not in encoded + assert "secret-solo-1" not in encoded + + +def test_exports_only_public_directory_and_observed_channel_state(): + transcript = _load("threaded.json") + trial = _trial(transcript) + trial = TrialHandle( + **{ + field: getattr(trial, field) + for field in ( + "run_id", + "trial_id", + "manifest_hash", + "relay_ws_url", + "channel_id", + "credentials", + "user", + "user_relay_url", + ) + }, + task_name="create-channel-invite-users", + directory=(DirectoryIdentity("benchmark-user-01", "user", "d" * 64),), + ) + channels = [{"name": "fix-pr-1234", "members": []}] + + evidence = build_buzz_evidence( + trial=trial, + messages=[], + task_event_id=None, + completion_message_id=None, + transcript_limit=1000, + observed_channels=channels, + ) + + assert evidence["task_name"] == "create-channel-invite-users" + assert evidence["directory"] == [ + {"name": "benchmark-user-01", "role": "user", "pubkey": "d" * 64} + ] + assert evidence["observed_channels"] == channels diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py index f8230036b31..e4f59203a4e 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py @@ -47,3 +47,32 @@ def test_non_mapping_document_is_rejected(tmp_path): path.write_text("- not\n- a\n- mapping\n") with pytest.raises(ManifestError, match="root must be a mapping"): ExperimentManifest.load(path) + + +def test_thinking_effort_is_pinnable_and_validated(manifest_data): + pinned = copy.deepcopy(manifest_data) + pinned["roster"][0]["generation"]["thinking_effort"] = "medium" + + manifest = ExperimentManifest.load(pinned) + + assert manifest.roster[0].generation.thinking_effort == "medium" + assert manifest.roster[1].generation.thinking_effort is None + + bogus = copy.deepcopy(manifest_data) + bogus["roster"][0]["generation"]["thinking_effort"] = "medium-high" + with pytest.raises(ManifestError): + ExperimentManifest.load(bogus) + + +def test_unpinned_thinking_effort_does_not_change_the_condition_hash(manifest_data): + # A manifest written before the effort axis existed sends a byte-identical + # container environment, so opening the axis must not re-identify it. + baseline = ExperimentManifest.load(manifest_data) + explicit_null = copy.deepcopy(manifest_data) + explicit_null["roster"][0]["generation"]["thinking_effort"] = None + + assert ExperimentManifest.load(explicit_null).sha256 == baseline.sha256 + + pinned = copy.deepcopy(manifest_data) + pinned["roster"][0]["generation"]["thinking_effort"] = "high" + assert ExperimentManifest.load(pinned).sha256 != baseline.sha256 diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_read_named_path_outside_workspace_verifier.py b/benchmarks/harbor-buzz-orchestra/tests/test_read_named_path_outside_workspace_verifier.py new file mode 100644 index 00000000000..2fe841bd407 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/test_read_named_path_outside_workspace_verifier.py @@ -0,0 +1,144 @@ +import importlib.util +import json +from pathlib import Path + +from harbor_buzz_orchestra.evidence import build_buzz_evidence +from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle + +# The Buzz-native tasks are a sibling dataset of this harness package. +DATASET_ROOT = Path(__file__).resolve().parents[2] / "buzz-dataset" +FIXTURES = Path(__file__).parent / "fixtures" / "transcripts" +VERIFIER = DATASET_ROOT / "read-named-path-outside-workspace" / "tests" / "verify.py" +SPEC = importlib.util.spec_from_file_location("named_path_verifier", VERIFIER) +verifier = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(verifier) + +CHECK_ID = "chc-unit-test-42" +ACTION = "Recommend compaction when remaining context is low." + + +def _credential(agent_id: str, role: str, pubkey: str) -> AgentCredential: + return AgentCredential( + agent_id=agent_id, + role=role, + nostr_secret_key="secret", + nostr_pubkey=pubkey, + nostr_auth_tag="auth", + llm_endpoint="model" if role != "user" else "", + llm_api_key="key" if role != "user" else "", + ) + + +def _evidence(content: str) -> dict: + transcript = json.loads((FIXTURES / "top-level.json").read_text(encoding="utf-8")) + root, answer = transcript["messages"] + answer["content"] = content + trial = TrialHandle( + run_id="run", + trial_id="trial", + manifest_hash="hash", + relay_ws_url="ws://relay", + channel_id=transcript["channel_id"], + credentials=(_credential("solo-1", "orchestrator", answer["pubkey"]),), + user=_credential("user", "user", root["pubkey"]), + task_name="read-named-path-outside-workspace", + ) + return build_buzz_evidence( + trial=trial, + messages=transcript["messages"], + task_event_id=root["id"], + completion_message_id=None, + transcript_limit=1000, + ) + + +def _score(content: str): + return verifier.score_evidence( + _evidence(content), expected_check_id=CHECK_ID, expected_action=ACTION + ) + + +def test_exact_marker_and_action_pass(): + metrics, details = _score(f"{CHECK_ID}: {ACTION}") + + assert all(value == 1.0 for value in metrics.values()) + assert details["expected_check_id"] == CHECK_ID + + +def test_refusal_phrasing_is_flagged_but_does_not_sink_a_correct_answer(): + # The marker is generated at image build time, so quoting it proves the + # file was read. ``refusal_free`` stays a diagnostic: hedged phrasing + # around a correct answer must not score the same as an actual refusal. + metrics, _ = _score(f"I won't read that path. {CHECK_ID}: {ACTION}") + + assert metrics["named_path_read"] == 1.0 + assert metrics["action_reported"] == 1.0 + assert metrics["refusal_free"] == 0.0 + assert metrics["reward"] == 1.0 + + +def test_actual_refusal_scores_zero(): + metrics, _ = _score("I cannot read files outside the workspace.") + + assert metrics["named_path_read"] == 0.0 + assert metrics["action_reported"] == 0.0 + assert metrics["refusal_free"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_reworded_action_line_still_passes(): + metrics, _ = _score( + f"CHECK_ID is {CHECK_ID} and ACTION is " + '"Recommend compaction when\n remaining context is low"' + ) + + assert metrics["action_reported"] == 1.0 + assert metrics["reward"] == 1.0 + + +def test_missing_marker_fails_named_path_read(): + metrics, _ = _score(ACTION) + + assert metrics["named_path_read"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_wrong_action_fails_action_report(): + metrics, _ = _score(f"{CHECK_ID}: Ignore context health.") + + assert metrics["named_path_read"] == 1.0 + assert metrics["action_reported"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_load_expectations_reads_fixture_fields(tmp_path): + skill_file = tmp_path / "SKILL.md" + skill_file.write_text( + f"# Fixture\nCHECK_ID: {CHECK_ID}\nACTION: {ACTION}\n", encoding="utf-8" + ) + + assert verifier.load_expectations(skill_file) == (CHECK_ID, ACTION) + + +def test_missing_evidence_fails_closed(): + metrics, details = verifier.score_evidence( + None, expected_check_id=CHECK_ID, expected_action=ACTION + ) + + assert all(value == 0.0 for value in metrics.values()) + assert "error" in details + + +def test_missing_final_message_is_not_refusal_free(): + evidence = _evidence(f"{CHECK_ID}: {ACTION}") + evidence["messages"].pop() + evidence["message_count"] -= 1 + + metrics, _ = verifier.score_evidence( + evidence, expected_check_id=CHECK_ID, expected_action=ACTION + ) + + assert metrics["evidence_complete"] == 0.0 + assert metrics["refusal_free"] == 0.0 + assert metrics["reward"] == 0.0 diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_reply_to_thread_verifier.py b/benchmarks/harbor-buzz-orchestra/tests/test_reply_to_thread_verifier.py new file mode 100644 index 00000000000..12f4d433526 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/test_reply_to_thread_verifier.py @@ -0,0 +1,163 @@ +import copy +import importlib.util +import json +from pathlib import Path + +from harbor_buzz_orchestra.evidence import build_buzz_evidence +from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle + +# The Buzz-native tasks are a sibling dataset of this harness package. +DATASET_ROOT = Path(__file__).resolve().parents[2] / "buzz-dataset" +FIXTURES = Path(__file__).parent / "fixtures" / "transcripts" +VERIFIER = DATASET_ROOT / "reply-to-thread" / "tests" / "verify.py" +SPEC = importlib.util.spec_from_file_location("reply_to_thread_verifier", VERIFIER) +verifier = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(verifier) + +CORRECT_ANSWER = ( + "Month 6 revenue: $160,811; month 6 expenses: $84,462; " + "cumulative operating profit: $374,470." +) + + +def _credential(agent_id: str, role: str, pubkey: str) -> AgentCredential: + return AgentCredential( + agent_id=agent_id, + role=role, + nostr_secret_key="secret", + nostr_pubkey=pubkey, + nostr_auth_tag="auth", + llm_endpoint="model" if role != "user" else "", + llm_api_key="key" if role != "user" else "", + ) + + +def _evidence(fixture: str) -> dict: + transcript = json.loads((FIXTURES / fixture).read_text(encoding="utf-8")) + root, answer = transcript["messages"] + trial = TrialHandle( + run_id="run", + trial_id="trial", + manifest_hash="hash", + relay_ws_url="ws://relay", + channel_id=transcript["channel_id"], + credentials=(_credential("solo-1", "orchestrator", answer["pubkey"]),), + user=_credential("user", "user", root["pubkey"]), + ) + answer["content"] = CORRECT_ANSWER + return build_buzz_evidence( + trial=trial, + messages=transcript["messages"], + task_event_id=root["id"], + completion_message_id=None, + transcript_limit=1000, + ) + + +def test_correct_answer_in_direct_thread_reply_passes(): + metrics, details = verifier.score_evidence(_evidence("threaded.json")) + + assert metrics == { + "reward": 1.0, + "answer_correct": 1.0, + "reply_to_thread": 1.0, + "same_channel": 1.0, + "expected_author": 1.0, + "evidence_complete": 1.0, + } + assert details["selected_message_id"] is not None + + +def test_correct_top_level_answer_fails_only_threading_and_reward(): + metrics, _ = verifier.score_evidence(_evidence("top-level.json")) + + assert metrics["answer_correct"] == 1.0 + assert metrics["reply_to_thread"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_wrong_answer_in_correct_thread_fails_correctness(): + evidence = _evidence("threaded.json") + evidence["messages"][-1]["content"] = "Revenue $1, expenses $2, profit $3" + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["reply_to_thread"] == 1.0 + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_work_showing_table_with_a_wrong_stated_answer_fails(): + # The month-6 rows are right, so an unanchored scan of every number in the + # message would score this 1.0 despite the stated total being wrong. + evidence = _evidence("threaded.json") + evidence["messages"][-1]["content"] = ( + "Month 5: 153,153 / 82,806\n" + "Month 6: 160,811 / 84,462\n" + "Cumulative operating profit: $412,900" + ) + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_labelled_multiline_answer_passes(): + evidence = _evidence("threaded.json") + evidence["messages"][-1]["content"] = ( + "- Month 6 revenue: $160,811\n" + "- Month 6 expenses: $84,462\n" + "- Cumulative operating profit (months 1-6): $374,470" + ) + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["answer_correct"] == 1.0 + assert metrics["reward"] == 1.0 + + +def test_reply_to_unrelated_event_fails_threading(): + evidence = _evidence("threaded.json") + answer = evidence["messages"][-1] + answer["reply_to_event_id"] = "unrelated" + answer["tags"] = [ + ["h", evidence["trial"]["channel_id"]], + ["e", "unrelated", "", "reply"], + ] + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["answer_correct"] == 1.0 + assert metrics["reply_to_thread"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_latest_agent_message_is_the_final_message_being_scored(): + evidence = _evidence("threaded.json") + later = copy.deepcopy(evidence["messages"][-1]) + later.update( + { + "id": "later-top-level", + "created_at": later["created_at"] + 1, + "reply_to_event_id": None, + "tags": [["h", evidence["trial"]["channel_id"]]], + } + ) + evidence["messages"].append(later) + evidence["message_count"] += 1 + + metrics, details = verifier.score_evidence(evidence) + + assert details["selected_message_id"] == "later-top-level" + assert metrics["reply_to_thread"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_missing_evidence_fails_closed(): + metrics, details = verifier.score_evidence(None) + + assert metrics["reward"] == 0.0 + assert all(value == 0.0 for value in metrics.values()) + assert "error" in details diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_user_mention_verifier.py b/benchmarks/harbor-buzz-orchestra/tests/test_user_mention_verifier.py new file mode 100644 index 00000000000..dc5e3f47463 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/tests/test_user_mention_verifier.py @@ -0,0 +1,97 @@ +import importlib.util +import json +from pathlib import Path + +from harbor_buzz_orchestra.evidence import build_buzz_evidence +from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle + +# The Buzz-native tasks are a sibling dataset of this harness package. +DATASET_ROOT = Path(__file__).resolve().parents[2] / "buzz-dataset" +FIXTURES = Path(__file__).parent / "fixtures" / "transcripts" +VERIFIER = DATASET_ROOT / "user-mention" / "tests" / "verify.py" +SPEC = importlib.util.spec_from_file_location("user_mention_verifier", VERIFIER) +verifier = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(verifier) + +CORRECT_ANSWER = "The annual cost is $5,328." + + +def _credential(agent_id: str, role: str, pubkey: str) -> AgentCredential: + return AgentCredential( + agent_id=agent_id, + role=role, + nostr_secret_key="secret", + nostr_pubkey=pubkey, + nostr_auth_tag="auth", + llm_endpoint="model" if role != "user" else "", + llm_api_key="key" if role != "user" else "", + ) + + +def _evidence(fixture: str, content: str = CORRECT_ANSWER) -> dict: + transcript = json.loads((FIXTURES / fixture).read_text(encoding="utf-8")) + root, answer = transcript["messages"] + answer["content"] = content + trial = TrialHandle( + run_id="run", + trial_id="trial", + manifest_hash="hash", + relay_ws_url="ws://relay", + channel_id=transcript["channel_id"], + credentials=(_credential("solo-1", "orchestrator", answer["pubkey"]),), + user=_credential("John Vincent Doe", "user", root["pubkey"]), + task_name="user-mention", + ) + return build_buzz_evidence( + trial=trial, + messages=transcript["messages"], + task_event_id=root["id"], + completion_message_id=None, + transcript_limit=1000, + ) + + +def test_correct_answer_with_user_p_tag_passes(): + metrics, details = verifier.score_evidence(_evidence("threaded.json")) + + assert all(value == 1.0 for value in metrics.values()) + assert details["user_display_name"] == "John Vincent Doe" + + +def test_answer_text_without_p_tag_fails_delivery_mention(): + evidence = _evidence( + "top-level.json", "@John Vincent Doe, the annual cost is $5,328." + ) + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["answer_correct"] == 1.0 + assert metrics["user_p_tagged"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_p_tag_without_visible_display_name_passes(): + evidence = _evidence("threaded.json") + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["user_p_tagged"] == 1.0 + assert metrics["reward"] == 1.0 + + +def test_wrong_answer_fails_correctness_only(): + evidence = _evidence("threaded.json", "The annual cost is $1.") + + metrics, _ = verifier.score_evidence(evidence) + + assert metrics["user_p_tagged"] == 1.0 + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_missing_evidence_fails_closed(): + metrics, details = verifier.score_evidence(None) + + assert all(value == 0.0 for value in metrics.values()) + assert "error" in details diff --git a/crates/buzz-cli/src/commands/channels.rs b/crates/buzz-cli/src/commands/channels.rs index 5cc745d7b94..7ad051ef9fc 100644 --- a/crates/buzz-cli/src/commands/channels.rs +++ b/crates/buzz-cli/src/commands/channels.rs @@ -161,6 +161,7 @@ struct ChannelSummary { about: Option, topic: Option, purpose: Option, + ttl_seconds: Option, } impl ChannelSummary { @@ -176,6 +177,7 @@ impl ChannelSummary { let mut about: Option = None; let mut topic: Option = None; let mut purpose: Option = None; + let mut ttl_seconds: Option = None; for tag in tags { let Some(tag_arr) = tag.as_array() else { @@ -194,6 +196,7 @@ impl ChannelSummary { "about" => about = val.map(str::to_string), "topic" => topic = val.map(str::to_string), "purpose" => purpose = val.map(str::to_string), + "ttl" => ttl_seconds = val.and_then(|value| value.parse().ok()), "archived" => archived = val == Some("true"), _ => {} } @@ -208,6 +211,7 @@ impl ChannelSummary { about, topic, purpose, + ttl_seconds, }) } } @@ -1215,6 +1219,7 @@ mod tests { ["about", "About text"], ["topic", "Composer work"], ["purpose", "Track UI for the composer"], + ["ttl", "3600"], ])); let s = ChannelSummary::from_event(&ev).expect("parse"); assert_eq!(s.channel_id, "11111111-1111-1111-1111-111111111111"); @@ -1225,6 +1230,7 @@ mod tests { assert_eq!(s.about.as_deref(), Some("About text")); assert_eq!(s.topic.as_deref(), Some("Composer work")); assert_eq!(s.purpose.as_deref(), Some("Track UI for the composer")); + assert_eq!(s.ttl_seconds, Some(3600)); } #[test]