fix: retrieving cold memories; add regression task

benchmarks/buzz-dataset/README.md

@@ -16,6 +16,7 @@ willing to read.
 | [`interleaved-agent-reports`](interleaved-agent-reports) | Workflow | Retains and synthesizes every report in a batch of agent messages |
 | [`cross-thread-requests`](cross-thread-requests) | Workflow | Keeps simultaneous top-level requests isolated and replies to both exact threads |
 | [`ambiguous-user-mention`](ambiguous-user-mention) | Workflow | Resolves duplicate display names and notifies only the intended pubkey |
+| [`memory-retrieval`](memory-retrieval) | Regression | Answers from harness-seeded cold memory without the value appearing in channel history |
 
 For `reply-to-thread` and `user-mention` the graded behavior is **deliberately
 absent from `instruction.md`** — it has to come from `buzz-acp`'s production

benchmarks/buzz-dataset/memory-retrieval/README.md

@@ -0,0 +1,18 @@
+# memory-retrieval
+
+Before the agent starts, the harness runs `buzz mem set` with the agent's own
+Buzz credentials to seed five similar cold memories. One records the exact
+total customer count for April 2024; the other four contain customer counts for
+nearby months or related April metrics. The harness then delivers
+`instruction.md`, which contains only the retrieval question and does not reveal
+the answer or memory slug. No channel message contains the answer, so
+conversation history cannot supply it.
+
+Full credit requires the exact customer count `352,345` in the threaded answer.
+Equivalent comma-free formatting is accepted, but rounded or approximate counts
+receive no credit. Credit is also voided if the answer mentions another number,
+apart from the requested year `2024`. This includes every count drawn from the
+distractor memories, so dumping several memories or selecting the wrong one does
+not pass — the answer must resolve to the correct value alone. The verifier does
+not inspect tool calls: seeding is deterministic harness setup, and retrieval is
+graded only through the observable answer.

benchmarks/buzz-dataset/memory-retrieval/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /app

benchmarks/buzz-dataset/memory-retrieval/instruction.md

@@ -0,0 +1 @@
+How many total customers did we have in April 2024?

benchmarks/buzz-dataset/memory-retrieval/task.toml

@@ -0,0 +1,25 @@
+schema_version = "1.3"
+
+[task]
+name = "buzz-native/memory-retrieval"
+description = "Answer a question using a harness-seeded cold-memory rule."
+authors = [{ name = "Buzz" }]
+keywords = ["buzz-native", "agents", "memory", "retrieval"]
+
+[metadata]
+evaluation_layer = "regression"
+difficulty = "hard"
+category = "collaboration"
+tags = ["agents", "memory", "retrieval"]
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 30.0
+
+[environment]
+network_mode = "public"
+cpus = 1
+memory_mb = 1024
+storage_mb = 1024

benchmarks/buzz-dataset/memory-retrieval/tests/test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+
+mkdir -p /logs/verifier
+python3 /tests/verify.py --evidence /logs/artifacts/buzz-evidence.json --reward /logs/verifier/reward.json --details /logs/verifier/details.json

benchmarks/buzz-dataset/memory-retrieval/tests/verify.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Deterministic verifier for pre-seeded cold-memory retrieval."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+EXPECTED_CUSTOMERS = 352_345
+ALLOWED_CONTEXT_NUMBERS = frozenset({2024})
+# Numbers that appear only in the distractor memories. Mentioning any of them
+# means the answer pulled from the wrong memory (or dumped several), so it does
+# not demonstrate that the correct value was selected.
+DISTRACTOR_NUMBERS = frozenset(
+    {
+        361_250,  # total-customers-per-month: monthly average
+        351_340,  # customer-value-metric: last month's customers
+        2_400,  # customer-value-metric: revenue per customer
+        325_401,  # customers-metrics-spring-24: March total
+        3_710,  # customers-metrics-spring-24: April active customers named John
+        21_604,  # new-customers-april-2024: April new customers
+    }
+)
+NUMBER = re.compile(r"(?<![A-Za-z0-9_])-?\d[\d,]*(?:\.\d+)?")
+
+
+def _zero() -> dict[str, float]:
+    return {
+        "reward": 0.0,
+        "answer_correct": 0.0,
+        "threaded_reply": 0.0,
+        "evidence_complete": 0.0,
+    }
+
+
+def _numbers(content: str) -> list[float]:
+    values: list[float] = []
+    for token in NUMBER.findall(content):
+        try:
+            values.append(float(token.replace(",", "")))
+        except ValueError:
+            continue
+    return values
+
+
+def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
+    if not isinstance(evidence, dict):
+        return _zero(), {"error": "evidence root is not an object"}
+
+    identities = evidence.get("identities", {})
+    agents = (
+        [
+            row
+            for row in identities.values()
+            if isinstance(row, dict) and row.get("role") == "orchestrator"
+        ]
+        if isinstance(identities, dict)
+        else []
+    )
+    agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None
+    question_id = evidence.get("task_event_id")
+    trial = evidence.get("trial", {})
+    question_channel = trial.get("channel_id") if isinstance(trial, dict) else None
+
+    messages = [row for row in evidence.get("messages", []) if isinstance(row, dict)]
+    replies = [
+        row
+        for row in messages
+        if agent_pubkey
+        and row.get("pubkey") == agent_pubkey
+        and row.get("channel_id") == question_channel
+        and row.get("reply_to_event_id") == question_id
+    ]
+    answer = replies[-1] if replies else None
+    content = str(answer.get("content", "")) if answer else ""
+    values = _numbers(content)
+    mentions_expected = any(value == EXPECTED_CUSTOMERS for value in values)
+    mentions_distractor = any(value in DISTRACTOR_NUMBERS for value in values)
+    noise_numbers = [
+        value
+        for value in values
+        if value != EXPECTED_CUSTOMERS and value not in ALLOWED_CONTEXT_NUMBERS
+    ]
+    answer_correct = float(mentions_expected and not noise_numbers)
+    threaded_reply = float(answer is not None)
+    evidence_complete = float(
+        evidence.get("schema_version") == 1
+        and evidence.get("task_name") == "memory-retrieval"
+        and evidence.get("truncated") is False
+        and len(agents) == 1
+        and isinstance(question_id, str)
+        and isinstance(question_channel, str)
+    )
+
+    structural_score = float(threaded_reply == 1.0 and evidence_complete == 1.0)
+    metrics = {
+        "reward": answer_correct * structural_score,
+        "answer_correct": answer_correct,
+        "threaded_reply": threaded_reply,
+        "evidence_complete": evidence_complete,
+    }
+    return metrics, {
+        "question_event_id": question_id,
+        "question_channel_id": question_channel,
+        "answer_message_id": answer.get("id") if answer else None,
+        "answer_content": content,
+        "parsed_numbers": values,
+        "expected_customers": EXPECTED_CUSTOMERS,
+        "mentions_expected": mentions_expected,
+        "mentions_distractor": mentions_distractor,
+        "noise_numbers": noise_numbers,
+        "allowed_context_numbers": sorted(ALLOWED_CONTEXT_NUMBERS),
+        "distractor_numbers": sorted(DISTRACTOR_NUMBERS),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--evidence", type=Path, required=True)
+    parser.add_argument("--reward", type=Path, required=True)
+    parser.add_argument("--details", type=Path, required=True)
+    args = parser.parse_args()
+    try:
+        metrics, details = score_evidence(
+            json.loads(args.evidence.read_text(encoding="utf-8"))
+        )
+    except (OSError, json.JSONDecodeError) as error:
+        metrics, details = _zero(), {"error": str(error)}
+    args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8")
+    args.details.write_text(
+        json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/harbor-buzz-orchestra/README.md

@@ -69,8 +69,8 @@ directory of this harness, not a subdirectory of it — scores Buzz product
 behavior alongside task correctness. It covers direct thread replies, callback
 user mentions, targeted reads of named paths, exact channel membership,
 multiline delivery, non-waking narrative names, batched reports, cross-thread
-isolation, and ambiguous identities. Run one task with the production base
-prompt from the checked-out source build:
+isolation, ambiguous identities, and explicit cold-memory retrieval. Run one
+task with the production base prompt from the checked-out source build:
 
 ```bash
 just benchmark \

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py

@@ -167,6 +167,8 @@ async def run(
                     "--name",
                     credential.agent_id,
                 )
+            await self._seed_memories(orchestrator, trial)
+            for credential in trial.credentials:
                 agents.append(
                     await self._launch_agent(
                         environment=environment,
@@ -786,6 +788,39 @@ async def _verify_m1_output(
                 f"and its stripped text must equal 'Hello, world!' ({detail})"
             )
 
+    async def _seed_memories(
+        self, credential: AgentCredential, trial: TrialHandle
+    ) -> None:
+        """Seed task-declared cold memory without exposing its value to the agent."""
+        for seed in fixture_for(trial.task_name).memory_seeds:
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    self.buzz_cli_binary,
+                    "mem",
+                    "set",
+                    seed.slug,
+                    "-",
+                    stdin=asyncio.subprocess.PIPE,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    env={
+                        **os.environ,
+                        "BUZZ_RELAY_URL": self._user_relay_url(trial),
+                        "BUZZ_PRIVATE_KEY": credential.nostr_secret_key,
+                        "BUZZ_AUTH_TAG": credential.nostr_auth_tag,
+                    },
+                )
+                _, stderr = await process.communicate(seed.value.encode())
+            except OSError as error:
+                raise RuntimeLaunchError(
+                    f"cannot seed cold memory {seed.slug!r}: {error}"
+                ) from None
+            if process.returncode != 0:
+                detail = stderr.decode(errors="replace").strip()
+                raise RuntimeLaunchError(
+                    f"buzz mem set {seed.slug} - exited {process.returncode}: {detail}"
+                )
+
     async def _send(
         self,
         credential: AgentCredential,

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py

@@ -32,6 +32,14 @@ class ScriptedMessage:
     mention_orchestrator: bool = True
 
 
+@dataclass(frozen=True, slots=True)
+class MemorySeed:
+    """A cold-memory value seeded under the orchestrator's identity."""
+
+    slug: str
+    value: str
+
+
 @dataclass(frozen=True, slots=True)
 class BuzzTaskFixture:
     """Relay state a task needs before the agent receives its prompt."""
@@ -40,6 +48,7 @@ class BuzzTaskFixture:
     scripted_messages: tuple[ScriptedMessage, ...] = ()
     observe_channel_names: tuple[str, ...] = ()
     user_display_name: str | None = None
+    memory_seeds: tuple[MemorySeed, ...] = ()
     # Whether the task's verifier grades the exported relay snapshot. Only
     # these tasks fail when the export fails; a Terminal-Bench task is graded
     # by its own tests and must not be errored by a snapshot hiccup.
@@ -59,6 +68,7 @@ class BuzzTaskFixture:
 INTERLEAVED_AGENT_REPORTS_TASK = "interleaved-agent-reports"
 CROSS_THREAD_REQUESTS_TASK = "cross-thread-requests"
 AMBIGUOUS_USER_MENTION_TASK = "ambiguous-user-mention"
+MEMORY_RETRIEVAL_TASK = "memory-retrieval"
 
 _CREATE_CHANNEL_FIXTURE = BuzzTaskFixture(
     directory=tuple(
@@ -161,6 +171,38 @@ class BuzzTaskFixture:
     requires_evidence=True,
 )
 
+
+# Noisy memories test retrieval of one relevant value through `buzz mem ls/get`.
+_MEMORY_RETRIEVAL_FIXTURE = BuzzTaskFixture(
+    user_display_name="Amelia Rose Bennett",
+    memory_seeds=(
+        MemorySeed(
+            slug="total-customers-per-month",
+            value="We average 361,250 customers per month.",
+        ),
+        MemorySeed(
+            slug="customer-value-metric",
+            value="Last month we had 351,340 customers with a $2400 revenue per customer",
+        ),
+        MemorySeed(
+            slug="customers-metrics-spring-24",
+            value=(
+                "In March, we had 325,401 total customers. In April, we had "
+                "3,710 active customers named John."
+            ),
+        ),
+        MemorySeed(
+            slug="new-customers-april-2024",
+            value="There are 21,604 new customers in April 2024.",
+        ),
+        MemorySeed(
+            slug="total-customers-metric",
+            value="In April 2024, we had 352,345 total customers.",
+        ),
+    ),
+    requires_evidence=True,
+)
+
 _FIXTURES = {
     CREATE_CHANNEL_TASK: _CREATE_CHANNEL_FIXTURE,
     USER_MENTION_TASK: _USER_MENTION_FIXTURE,
@@ -173,6 +215,7 @@ class BuzzTaskFixture:
     INTERLEAVED_AGENT_REPORTS_TASK: _INTERLEAVED_AGENT_REPORTS_FIXTURE,
     CROSS_THREAD_REQUESTS_TASK: _CROSS_THREAD_REQUESTS_FIXTURE,
     AMBIGUOUS_USER_MENTION_TASK: _AMBIGUOUS_USER_MENTION_FIXTURE,
+    MEMORY_RETRIEVAL_TASK: _MEMORY_RETRIEVAL_FIXTURE,
 }
 
 

benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md

@@ -9,6 +9,13 @@ endpoint string remains the join key.
 Every key in these files must be a manifest endpoint name; the loader treats
 all entries as endpoint configs (no comment keys).
 
+## openai-live-wire-debug.json
+
+Diagnostic variant of `openai-live.json` for local runs. It enables
+`acp::wire=debug`, so retained agent stdout logs include full ACP messages,
+including tool-call arguments and results. These logs may contain prompt or
+command content; keep them local. The verifier and reward do not read them.
+
 ## m1-local.json
 
 M1 wiring proof: both placeholder endpoints resolve to one local llama-server

benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live-wire-debug.json

@@ -0,0 +1,9 @@
+{
+  "gpt-5.6-luna": {
+    "provider": "openai",
+    "api_key_env": "OPENAI_COMPAT_API_KEY",
+    "env": {
+      "RUST_LOG": "acp::wire=debug"
+    }
+  }
+}

benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py

@@ -77,6 +77,7 @@ def test_buzz_task_metadata_defines_the_expected_layers():
             "user-mention",
             "read-named-path-outside-workspace",
             "multiline-message",
+            "memory-retrieval",
             "narrative-agent-names",
         },
         "workflow": {
@@ -167,7 +168,7 @@ def test_explicit_attempts_override_keeps_one_mixed_buzz_job():
     (run,) = benchmark.plan_benchmark_runs(args)
 
     assert run.attempts == 7
-    assert len(run.include_task) == 9
+    assert len(run.include_task) == 10
 
     layered = benchmark.parse_args(
         [