diff --git a/benchmarks/buzz-dataset/README.md b/benchmarks/buzz-dataset/README.md index cfce3b257a2..6df3fbadcf3 100644 --- a/benchmarks/buzz-dataset/README.md +++ b/benchmarks/buzz-dataset/README.md @@ -16,6 +16,7 @@ willing to read. | [`interleaved-agent-reports`](interleaved-agent-reports) | Workflow | Retains and synthesizes every report in a batch of agent messages | | [`cross-thread-requests`](cross-thread-requests) | Workflow | Keeps simultaneous top-level requests isolated and replies to both exact threads | | [`ambiguous-user-mention`](ambiguous-user-mention) | Workflow | Resolves duplicate display names and notifies only the intended pubkey | +| [`memory-retrieval`](memory-retrieval) | Regression | Answers from harness-seeded cold memory without the value appearing in channel history | For `reply-to-thread` and `user-mention` the graded behavior is **deliberately absent from `instruction.md`** — it has to come from `buzz-acp`'s production diff --git a/benchmarks/buzz-dataset/memory-retrieval/README.md b/benchmarks/buzz-dataset/memory-retrieval/README.md new file mode 100644 index 00000000000..ef6e63950f2 --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/README.md @@ -0,0 +1,18 @@ +# memory-retrieval + +Before the agent starts, the harness runs `buzz mem set` with the agent's own +Buzz credentials to seed five similar cold memories. One records the exact +total customer count for April 2024; the other four contain customer counts for +nearby months or related April metrics. The harness then delivers +`instruction.md`, which contains only the retrieval question and does not reveal +the answer or memory slug. No channel message contains the answer, so +conversation history cannot supply it. + +Full credit requires the exact customer count `352,345` in the threaded answer. +Equivalent comma-free formatting is accepted, but rounded or approximate counts +receive no credit. Credit is also voided if the answer mentions another number, +apart from the requested year `2024`. This includes every count drawn from the +distractor memories, so dumping several memories or selecting the wrong one does +not pass — the answer must resolve to the correct value alone. The verifier does +not inspect tool calls: seeding is deterministic harness setup, and retrieval is +graded only through the observable answer. diff --git a/benchmarks/buzz-dataset/memory-retrieval/environment/Dockerfile b/benchmarks/buzz-dataset/memory-retrieval/environment/Dockerfile new file mode 100644 index 00000000000..29f16f3c412 --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /app diff --git a/benchmarks/buzz-dataset/memory-retrieval/instruction.md b/benchmarks/buzz-dataset/memory-retrieval/instruction.md new file mode 100644 index 00000000000..0a7a96173d5 --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/instruction.md @@ -0,0 +1 @@ +How many total customers did we have in April 2024? diff --git a/benchmarks/buzz-dataset/memory-retrieval/task.toml b/benchmarks/buzz-dataset/memory-retrieval/task.toml new file mode 100644 index 00000000000..a018303ce58 --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/task.toml @@ -0,0 +1,25 @@ +schema_version = "1.3" + +[task] +name = "buzz-native/memory-retrieval" +description = "Answer a question using a harness-seeded cold-memory rule." +authors = [{ name = "Buzz" }] +keywords = ["buzz-native", "agents", "memory", "retrieval"] + +[metadata] +evaluation_layer = "regression" +difficulty = "hard" +category = "collaboration" +tags = ["agents", "memory", "retrieval"] + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 30.0 + +[environment] +network_mode = "public" +cpus = 1 +memory_mb = 1024 +storage_mb = 1024 diff --git a/benchmarks/buzz-dataset/memory-retrieval/tests/test.sh b/benchmarks/buzz-dataset/memory-retrieval/tests/test.sh new file mode 100755 index 00000000000..79434035e3c --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/tests/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +mkdir -p /logs/verifier +python3 /tests/verify.py --evidence /logs/artifacts/buzz-evidence.json --reward /logs/verifier/reward.json --details /logs/verifier/details.json diff --git a/benchmarks/buzz-dataset/memory-retrieval/tests/verify.py b/benchmarks/buzz-dataset/memory-retrieval/tests/verify.py new file mode 100755 index 00000000000..e3ffe9f5c6a --- /dev/null +++ b/benchmarks/buzz-dataset/memory-retrieval/tests/verify.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Deterministic verifier for pre-seeded cold-memory retrieval.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +EXPECTED_CUSTOMERS = 352_345 +ALLOWED_CONTEXT_NUMBERS = frozenset({2024}) +# Numbers that appear only in the distractor memories. Mentioning any of them +# means the answer pulled from the wrong memory (or dumped several), so it does +# not demonstrate that the correct value was selected. +DISTRACTOR_NUMBERS = frozenset( + { + 361_250, # total-customers-per-month: monthly average + 351_340, # customer-value-metric: last month's customers + 2_400, # customer-value-metric: revenue per customer + 325_401, # customers-metrics-spring-24: March total + 3_710, # customers-metrics-spring-24: April active customers named John + 21_604, # new-customers-april-2024: April new customers + } +) +NUMBER = re.compile(r"(? dict[str, float]: + return { + "reward": 0.0, + "answer_correct": 0.0, + "threaded_reply": 0.0, + "evidence_complete": 0.0, + } + + +def _numbers(content: str) -> list[float]: + values: list[float] = [] + for token in NUMBER.findall(content): + try: + values.append(float(token.replace(",", ""))) + except ValueError: + continue + return values + + +def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]: + if not isinstance(evidence, dict): + return _zero(), {"error": "evidence root is not an object"} + + identities = evidence.get("identities", {}) + agents = ( + [ + row + for row in identities.values() + if isinstance(row, dict) and row.get("role") == "orchestrator" + ] + if isinstance(identities, dict) + else [] + ) + agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None + question_id = evidence.get("task_event_id") + trial = evidence.get("trial", {}) + question_channel = trial.get("channel_id") if isinstance(trial, dict) else None + + messages = [row for row in evidence.get("messages", []) if isinstance(row, dict)] + replies = [ + row + for row in messages + if agent_pubkey + and row.get("pubkey") == agent_pubkey + and row.get("channel_id") == question_channel + and row.get("reply_to_event_id") == question_id + ] + answer = replies[-1] if replies else None + content = str(answer.get("content", "")) if answer else "" + values = _numbers(content) + mentions_expected = any(value == EXPECTED_CUSTOMERS for value in values) + mentions_distractor = any(value in DISTRACTOR_NUMBERS for value in values) + noise_numbers = [ + value + for value in values + if value != EXPECTED_CUSTOMERS and value not in ALLOWED_CONTEXT_NUMBERS + ] + answer_correct = float(mentions_expected and not noise_numbers) + threaded_reply = float(answer is not None) + evidence_complete = float( + evidence.get("schema_version") == 1 + and evidence.get("task_name") == "memory-retrieval" + and evidence.get("truncated") is False + and len(agents) == 1 + and isinstance(question_id, str) + and isinstance(question_channel, str) + ) + + structural_score = float(threaded_reply == 1.0 and evidence_complete == 1.0) + metrics = { + "reward": answer_correct * structural_score, + "answer_correct": answer_correct, + "threaded_reply": threaded_reply, + "evidence_complete": evidence_complete, + } + return metrics, { + "question_event_id": question_id, + "question_channel_id": question_channel, + "answer_message_id": answer.get("id") if answer else None, + "answer_content": content, + "parsed_numbers": values, + "expected_customers": EXPECTED_CUSTOMERS, + "mentions_expected": mentions_expected, + "mentions_distractor": mentions_distractor, + "noise_numbers": noise_numbers, + "allowed_context_numbers": sorted(ALLOWED_CONTEXT_NUMBERS), + "distractor_numbers": sorted(DISTRACTOR_NUMBERS), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--reward", type=Path, required=True) + parser.add_argument("--details", type=Path, required=True) + args = parser.parse_args() + try: + metrics, details = score_evidence( + json.loads(args.evidence.read_text(encoding="utf-8")) + ) + except (OSError, json.JSONDecodeError) as error: + metrics, details = _zero(), {"error": str(error)} + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") + args.details.write_text( + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/harbor-buzz-orchestra/README.md b/benchmarks/harbor-buzz-orchestra/README.md index df859b41ab5..a0bd08a6ae0 100644 --- a/benchmarks/harbor-buzz-orchestra/README.md +++ b/benchmarks/harbor-buzz-orchestra/README.md @@ -69,8 +69,8 @@ directory of this harness, not a subdirectory of it — scores Buzz product behavior alongside task correctness. It covers direct thread replies, callback user mentions, targeted reads of named paths, exact channel membership, multiline delivery, non-waking narrative names, batched reports, cross-thread -isolation, and ambiguous identities. Run one task with the production base -prompt from the checked-out source build: +isolation, ambiguous identities, and explicit cold-memory retrieval. Run one +task with the production base prompt from the checked-out source build: ```bash just benchmark \ diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 797e3a860c2..a29b86e7314 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -167,6 +167,8 @@ async def run( "--name", credential.agent_id, ) + await self._seed_memories(orchestrator, trial) + for credential in trial.credentials: agents.append( await self._launch_agent( environment=environment, @@ -786,6 +788,39 @@ async def _verify_m1_output( f"and its stripped text must equal 'Hello, world!' ({detail})" ) + async def _seed_memories( + self, credential: AgentCredential, trial: TrialHandle + ) -> None: + """Seed task-declared cold memory without exposing its value to the agent.""" + for seed in fixture_for(trial.task_name).memory_seeds: + try: + process = await asyncio.create_subprocess_exec( + self.buzz_cli_binary, + "mem", + "set", + seed.slug, + "-", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env={ + **os.environ, + "BUZZ_RELAY_URL": self._user_relay_url(trial), + "BUZZ_PRIVATE_KEY": credential.nostr_secret_key, + "BUZZ_AUTH_TAG": credential.nostr_auth_tag, + }, + ) + _, stderr = await process.communicate(seed.value.encode()) + except OSError as error: + raise RuntimeLaunchError( + f"cannot seed cold memory {seed.slug!r}: {error}" + ) from None + if process.returncode != 0: + detail = stderr.decode(errors="replace").strip() + raise RuntimeLaunchError( + f"buzz mem set {seed.slug} - exited {process.returncode}: {detail}" + ) + async def _send( self, credential: AgentCredential, diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py index 451b97f9c12..87cffbbcf2c 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py @@ -32,6 +32,14 @@ class ScriptedMessage: mention_orchestrator: bool = True +@dataclass(frozen=True, slots=True) +class MemorySeed: + """A cold-memory value seeded under the orchestrator's identity.""" + + slug: str + value: str + + @dataclass(frozen=True, slots=True) class BuzzTaskFixture: """Relay state a task needs before the agent receives its prompt.""" @@ -40,6 +48,7 @@ class BuzzTaskFixture: scripted_messages: tuple[ScriptedMessage, ...] = () observe_channel_names: tuple[str, ...] = () user_display_name: str | None = None + memory_seeds: tuple[MemorySeed, ...] = () # Whether the task's verifier grades the exported relay snapshot. Only # these tasks fail when the export fails; a Terminal-Bench task is graded # by its own tests and must not be errored by a snapshot hiccup. @@ -59,6 +68,7 @@ class BuzzTaskFixture: INTERLEAVED_AGENT_REPORTS_TASK = "interleaved-agent-reports" CROSS_THREAD_REQUESTS_TASK = "cross-thread-requests" AMBIGUOUS_USER_MENTION_TASK = "ambiguous-user-mention" +MEMORY_RETRIEVAL_TASK = "memory-retrieval" _CREATE_CHANNEL_FIXTURE = BuzzTaskFixture( directory=tuple( @@ -161,6 +171,38 @@ class BuzzTaskFixture: requires_evidence=True, ) + +# Noisy memories test retrieval of one relevant value through `buzz mem ls/get`. +_MEMORY_RETRIEVAL_FIXTURE = BuzzTaskFixture( + user_display_name="Amelia Rose Bennett", + memory_seeds=( + MemorySeed( + slug="total-customers-per-month", + value="We average 361,250 customers per month.", + ), + MemorySeed( + slug="customer-value-metric", + value="Last month we had 351,340 customers with a $2400 revenue per customer", + ), + MemorySeed( + slug="customers-metrics-spring-24", + value=( + "In March, we had 325,401 total customers. In April, we had " + "3,710 active customers named John." + ), + ), + MemorySeed( + slug="new-customers-april-2024", + value="There are 21,604 new customers in April 2024.", + ), + MemorySeed( + slug="total-customers-metric", + value="In April 2024, we had 352,345 total customers.", + ), + ), + requires_evidence=True, +) + _FIXTURES = { CREATE_CHANNEL_TASK: _CREATE_CHANNEL_FIXTURE, USER_MENTION_TASK: _USER_MENTION_FIXTURE, @@ -173,6 +215,7 @@ class BuzzTaskFixture: INTERLEAVED_AGENT_REPORTS_TASK: _INTERLEAVED_AGENT_REPORTS_FIXTURE, CROSS_THREAD_REQUESTS_TASK: _CROSS_THREAD_REQUESTS_FIXTURE, AMBIGUOUS_USER_MENTION_TASK: _AMBIGUOUS_USER_MENTION_FIXTURE, + MEMORY_RETRIEVAL_TASK: _MEMORY_RETRIEVAL_FIXTURE, } diff --git a/benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md index f8d2a560bf2..a4a460d7087 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md +++ b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md @@ -9,6 +9,13 @@ endpoint string remains the join key. Every key in these files must be a manifest endpoint name; the loader treats all entries as endpoint configs (no comment keys). +## openai-live-wire-debug.json + +Diagnostic variant of `openai-live.json` for local runs. It enables +`acp::wire=debug`, so retained agent stdout logs include full ACP messages, +including tool-call arguments and results. These logs may contain prompt or +command content; keep them local. The verifier and reward do not read them. + ## m1-local.json M1 wiring proof: both placeholder endpoints resolve to one local llama-server diff --git a/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live-wire-debug.json b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live-wire-debug.json new file mode 100644 index 00000000000..0403481648d --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live-wire-debug.json @@ -0,0 +1,9 @@ +{ + "gpt-5.6-luna": { + "provider": "openai", + "api_key_env": "OPENAI_COMPAT_API_KEY", + "env": { + "RUST_LOG": "acp::wire=debug" + } + } +} diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py index 0644df63b36..de91726e426 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py @@ -77,6 +77,7 @@ def test_buzz_task_metadata_defines_the_expected_layers(): "user-mention", "read-named-path-outside-workspace", "multiline-message", + "memory-retrieval", "narrative-agent-names", }, "workflow": { @@ -167,7 +168,7 @@ def test_explicit_attempts_override_keeps_one_mixed_buzz_job(): (run,) = benchmark.plan_benchmark_runs(args) assert run.attempts == 7 - assert len(run.include_task) == 9 + assert len(run.include_task) == 10 layered = benchmark.parse_args( [ diff --git a/benchmarks/harbor-buzz-orchestra/testbed/uv.lock b/benchmarks/harbor-buzz-orchestra/testbed/uv.lock index 814f4d3527e..543499b3159 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/uv.lock +++ b/benchmarks/harbor-buzz-orchestra/testbed/uv.lock @@ -717,7 +717,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2" }, { name = "pyyaml", specifier = ">=6.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.3" }, ] provides-extras = ["dev"] @@ -743,7 +743,7 @@ requires-dist = [ { name = "harbor-buzz-orchestra", editable = "../" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.3" }, ] provides-extras = ["dev"] @@ -2026,27 +2026,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.20" -source = { registry = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/simple" } -sdist = { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566" } -wheels = [ - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca" }, +version = "0.16.3" +source = { registry = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/simple" } +sdist = { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2" } +wheels = [ + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a" }, ] [[package]] diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index 182db9893f6..ecdc9e4cdec 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -1,5 +1,6 @@ """The container runtime must launch the production stack, unmodified.""" +import asyncio import hashlib import json import re @@ -343,6 +344,56 @@ async def test_launch_wires_the_desktop_environment(tmp_path, configured, expect ) +def test_memory_task_disables_auto_memory_injection(tmp_path): + manifest = write_manifest(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = replace(trial_handle((orch,)), task_name="memory-retrieval") + + env = runtime(tmp_path)._agent_env( + trial=trial, + credential=orch, + agent_class=manifest.roster[0], + endpoint=EndpointLaunchConfig("anthropic", "ANTHROPIC_API_KEY"), + remote_prompt="/prompt.md", + ) + + assert env["BUZZ_ACP_CHANNELS"] == "channel" + assert env["BUZZ_ACP_NO_MEMORY"] == "true" + + +@pytest.mark.asyncio +async def test_memory_seed_uses_agent_credentials_and_stdin(tmp_path, monkeypatch): + orch = credential("orch-1", "orchestrator", "orch-model") + trial = replace(trial_handle((orch,)), task_name="memory-retrieval") + captured = [] + + class Process: + def __init__(self, invocation): + self.invocation = invocation + + returncode = 0 + + async def communicate(self, value): + self.invocation["value"] = value + return b"", b"wrote memory" + + async def create_subprocess_exec(*args, **kwargs): + invocation = {"args": args, "env": kwargs["env"]} + captured.append(invocation) + return Process(invocation) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess_exec) + + await runtime(tmp_path)._seed_memories(orch, trial) + + seeds = fixture_for("memory-retrieval").memory_seeds + assert len(captured) == len(seeds) + for invocation, seed in zip(captured, seeds, strict=True): + assert invocation["args"][1:] == ("mem", "set", seed.slug, "-") + assert invocation["env"]["BUZZ_PRIVATE_KEY"] == orch.nostr_secret_key + assert invocation["value"] == seed.value.encode() + + def test_runtime_validates_construction_bounds(tmp_path): # 0 is legal and means unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial # budget is the clock. Only negatives are rejected. diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_expanded_buzz_native_verifiers.py b/benchmarks/harbor-buzz-orchestra/tests/test_expanded_buzz_native_verifiers.py index 225cb1d1fa1..f39da50a3b7 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_expanded_buzz_native_verifiers.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_expanded_buzz_native_verifiers.py @@ -6,6 +6,8 @@ from pathlib import Path from types import ModuleType +from harbor_buzz_orchestra.task_fixtures import fixture_for + DATASET_ROOT = Path(__file__).resolve().parents[2] / "buzz-dataset" AGENT = "a" * 64 USER = "u" * 64 @@ -236,3 +238,102 @@ def test_ambiguous_user_mention_targets_only_profile_match(): metrics, _ = verifier.score_evidence(evidence) assert metrics["other_not_notified"] == 0.0 assert metrics["reward"] == 0.0 + + +def test_memory_retrieval_requires_correct_threaded_answer(): + verifier = _verifier("memory-retrieval") + evidence = _base("memory-retrieval", "Amelia Rose Bennett") + question_id = "memory-question" + evidence["task_event_id"] = question_id + answer = _message( + "answer", + "We had 352,345 total customers in April 2024.", + reply_to=question_id, + mentions=[USER], + ) + evidence["messages"] = [answer] + + for correct_answer in ( + "352,345", + "We had 352,345 total customers in April 2024.", + "April 2024 total customers: 352345", + ): + evidence["messages"][0]["content"] = correct_answer + metrics, _ = verifier.score_evidence(evidence) + assert all(value == 1.0 for value in metrics.values()) + + for answer_without_exact_total in ( + "361,250", + "351,340", + "$2,400 revenue per customer", + "325,401", + "3,710", + "21,604", + "352,344", + "352,346", + "352,000", + "About 352 thousand", + "Approximately 352.3 thousand", + ): + evidence["messages"][0]["content"] = answer_without_exact_total + metrics, _ = verifier.score_evidence(evidence) + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + for distractor in verifier.DISTRACTOR_NUMBERS: + evidence["messages"][0]["content"] = ( + f"We had 352,345 total customers. Another relevant count was {distractor:,}." + ) + metrics, details = verifier.score_evidence(evidence) + assert details["mentions_expected"] is True + assert details["mentions_distractor"] is True + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + for noise_count in (352_000, 999_999): + evidence["messages"][0]["content"] = ( + f"We had 352,345 total customers, approximately {noise_count:,}." + ) + metrics, details = verifier.score_evidence(evidence) + assert details["mentions_expected"] is True + assert details["mentions_distractor"] is False + assert details["noise_numbers"] == [float(noise_count)] + assert metrics["answer_correct"] == 0.0 + assert metrics["reward"] == 0.0 + + evidence["messages"][0]["content"] = "352,345" + evidence["messages"][0]["reply_to_event_id"] = "wrong-question" + metrics, _ = verifier.score_evidence(evidence) + assert metrics["answer_correct"] == 0.0 + assert metrics["threaded_reply"] == 0.0 + assert metrics["reward"] == 0.0 + + +def test_memory_retrieval_answer_exists_only_in_harness_seed(): + verifier = _verifier("memory-retrieval") + fixture = fixture_for("memory-retrieval") + instruction = (DATASET_ROOT / "memory-retrieval" / "instruction.md").read_text( + encoding="utf-8" + ) + + seeds = {seed.slug: seed.value for seed in fixture.memory_seeds} + assert set(seeds) == { + "total-customers-per-month", + "customer-value-metric", + "customers-metrics-spring-24", + "new-customers-april-2024", + "total-customers-metric", + } + assert "352,345" in seeds["total-customers-metric"] + assert sum("352,345" in value for value in seeds.values()) == 1 + seeded_distractors = frozenset( + number + for slug, value in seeds.items() + if slug != "total-customers-metric" + for number in verifier._numbers(value) + if number != 2024 + ) + assert verifier.EXPECTED_CUSTOMERS == 352_345 + assert verifier.DISTRACTOR_NUMBERS == seeded_distractors + assert "352,345" not in instruction + assert "352345" not in instruction.replace(",", "") diff --git a/benchmarks/harbor-buzz-orchestra/uv.lock b/benchmarks/harbor-buzz-orchestra/uv.lock index 05072d81a80..67b6390f365 100644 --- a/benchmarks/harbor-buzz-orchestra/uv.lock +++ b/benchmarks/harbor-buzz-orchestra/uv.lock @@ -696,7 +696,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2" }, { name = "pyyaml", specifier = ">=6.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15" }, + { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.3" }, ] provides-extras = ["dev"] @@ -1934,27 +1934,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.20" -source = { registry = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/simple" } -sdist = { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566" } -wheels = [ - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415" }, - { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca" }, +version = "0.16.3" +source = { registry = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/simple" } +sdist = { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2" } +wheels = [ + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948" }, + { url = "https://global.block-artifacts.com/artifactory/api/pypi/block-pypi/packages/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a" }, ] [[package]] diff --git a/crates/buzz-acp/src/base_prompt.md b/crates/buzz-acp/src/base_prompt.md index 4dc4720ed85..661af502204 100644 --- a/crates/buzz-acp/src/base_prompt.md +++ b/crates/buzz-acp/src/base_prompt.md @@ -27,6 +27,7 @@ The `buzz` CLI is your primary interface. Auth env vars: `BUZZ_RELAY_URL`, `BUZZ | `buzz issues` | `create`, `get`, `list`, `status`, `assign` | | `buzz pr` | `open`, `update`, `get`, `list`, `status` | | `buzz upload` | `file` | +| `buzz mem` | `set`, `get`, `ls`, `patch`, `rm` | Run `buzz --help` or `buzz --help` for full usage. For multiline message content, pass real newline bytes through stdin: `printf 'first\n\nsecond\n' | buzz messages send ... --content -`. Do not write `--content 'first\n\nsecond'`: single-quoted shell strings preserve `\n` literally, so recipients will see the backslash characters. `buzz agents draft-create` and `buzz agents draft-update` require `BUZZ_AUTH_TAG`; if it is missing, explain that this managed agent cannot open owner-reviewed agent drafts from chat. @@ -118,10 +119,11 @@ Do not discover, fetch, load, read, or use relay-backed skills unless the author Your `core` memory is auto-injected into your context every turn — it holds identity, durable rules, and goals across sessions. - **Keep `core` small.** A line earns a permanent slot only if it matters across most sessions or prevents a sharp repeat mistake. Treat the 65,535-byte hard limit as a wall to stay far from, not a budget to fill — aim to keep `core` under ~10 KB (roughly your healthy baseline). -- **Turn mistakes into durable lessons.** When a mistake exposes a repeatable mechanism, record the invariant in the same session. Keep only the load-bearing rule in `core`; put detailed evidence and procedures in cold memory. If the lesson improves a shared workflow, update the team's shared guidance so others do not have to re-earn it. -- **Durable detail goes to a cold `mem/` slug, not `core`.** Long-lived findings that don't need to be in front of you every turn belong in a `mem/` slug you read on demand — not appended to `core`. -- **Evict completed work.** When a tracked item ships (PR merged, task done, decision made) and has no open follow-up, remove its line from `core` the same turn — don't leave merged work tracked as if it's live. The detail already lives in its cold `mem/` slug if you need it later. +- **Turn mistakes into durable lessons.** When a mistake exposes a repeatable mechanism, record the invariant in the same session. Keep only the load-bearing rule in `core`; put detailed evidence and procedures in cold memory with `buzz mem set`. If the lesson improves a shared workflow, update the team's shared guidance so others do not have to re-earn it. +- **Durable detail goes to a cold `buzz mem set `, not `core`.** Long-lived findings that don't need to be in front of you every turn belong in cold memory you read on demand with `buzz mem get `—not appended to `core`. +- **Evict completed work.** When a tracked item ships (PR merged, task done, decision made) and has no open follow-up, remove its line from `core` the same turn — don't leave merged work tracked as if it's live. The detail already lives in its cold `buzz mem` slug if you need it later. Always ask the owner before doing this. - **Treat `core` as load-bearing.** Follow it unless newer explicit user instructions override it. +- **Cold memory search and hygiene.** Find cold memory with `buzz mem ls` and `buzz mem get`. If a user's prompt contradicts a memory, always ask the owner if they would remove it with `buzz mem rm` or update it with `buzz mem patch`. Never remove or patch a memory without owner approval. - Cite sources with paths, links, or command outputs. No unsupported claims. ## Engineering Discipline