fix: retrieving cold memories; add regression task

benchmarks/buzz-dataset/README.md

@@ -16,6 +16,7 @@ willing to read.
 | [`interleaved-agent-reports`](interleaved-agent-reports) | Workflow | Retains and synthesizes every report in a batch of agent messages |
 | [`cross-thread-requests`](cross-thread-requests) | Workflow | Keeps simultaneous top-level requests isolated and replies to both exact threads |
 | [`ambiguous-user-mention`](ambiguous-user-mention) | Workflow | Resolves duplicate display names and notifies only the intended pubkey |
+| [`memory-retrieval`](memory-retrieval) | Regression | Answers from harness-seeded cold memory without the value appearing in channel history |
 
 For `reply-to-thread` and `user-mention` the graded behavior is **deliberately
 absent from `instruction.md`** — it has to come from `buzz-acp`'s production

benchmarks/buzz-dataset/memory-retrieval/README.md

@@ -0,0 +1,11 @@
+# memory-retrieval
+
+Before the agent starts, the harness runs `buzz mem set` with the agent's own
+Buzz credentials to seed a GPV rule as cold memory. It then delivers
+`instruction.md`, which contains only the retrieval question and does not
+reveal the memory value or slug. No channel message contains the answer, so
+conversation history cannot supply it.
+
+Passing requires a threaded answer that selects `net_gpv`. The verifier does
+not inspect tool calls: seeding is deterministic harness setup, and retrieval
+is graded only through the observable answer.

benchmarks/buzz-dataset/memory-retrieval/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /app

benchmarks/buzz-dataset/memory-retrieval/instruction.md

@@ -0,0 +1 @@
+What column should be used for gpv calculation?

benchmarks/buzz-dataset/memory-retrieval/task.toml

@@ -0,0 +1,25 @@
+schema_version = "1.3"
+
+[task]
+name = "buzz-native/memory-retrieval"
+description = "Answer a question using a harness-seeded cold-memory rule."
+authors = [{ name = "Buzz" }]
+keywords = ["buzz-native", "agents", "memory", "retrieval"]
+
+[metadata]
+evaluation_layer = "regression"
+difficulty = "hard"
+category = "collaboration"
+tags = ["agents", "memory", "retrieval"]
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 30.0
+
+[environment]
+network_mode = "public"
+cpus = 1
+memory_mb = 1024
+storage_mb = 1024

benchmarks/buzz-dataset/memory-retrieval/tests/test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+
+mkdir -p /logs/verifier
+python3 /tests/verify.py --evidence /logs/artifacts/buzz-evidence.json --reward /logs/verifier/reward.json --details /logs/verifier/details.json

benchmarks/buzz-dataset/memory-retrieval/tests/verify.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Deterministic verifier for pre-seeded cold-memory retrieval."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+def _zero() -> dict[str, float]:
+    return {
+        "reward": 0.0,
+        "answer_correct": 0.0,
+        "threaded_reply": 0.0,
+        "evidence_complete": 0.0,
+    }
+
+
+def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
+    if not isinstance(evidence, dict):
+        return _zero(), {"error": "evidence root is not an object"}
+
+    identities = evidence.get("identities", {})
+    agents = (
+        [
+            row
+            for row in identities.values()
+            if isinstance(row, dict) and row.get("role") == "orchestrator"
+        ]
+        if isinstance(identities, dict)
+        else []
+    )
+    agent_pubkey = agents[0].get("pubkey") if len(agents) == 1 else None
+    question_id = evidence.get("task_event_id")
+    trial = evidence.get("trial", {})
+    question_channel = trial.get("channel_id") if isinstance(trial, dict) else None
+
+    messages = [row for row in evidence.get("messages", []) if isinstance(row, dict)]
+    replies = [
+        row
+        for row in messages
+        if agent_pubkey
+        and row.get("pubkey") == agent_pubkey
+        and row.get("channel_id") == question_channel
+        and row.get("reply_to_event_id") == question_id
+    ]
+    answer = replies[-1] if replies else None
+    content = str(answer.get("content", "")) if answer else ""
+    answer_correct = float(bool(re.search(r"\bnet_gpv\b", content, re.IGNORECASE)))
+    threaded_reply = float(answer is not None)
+    evidence_complete = float(
+        evidence.get("schema_version") == 1
+        and evidence.get("task_name") == "memory-retrieval"
+        and evidence.get("truncated") is False
+        and len(agents) == 1
+        and isinstance(question_id, str)
+        and isinstance(question_channel, str)
+    )
+
+    values = (
+        answer_correct,
+        threaded_reply,
+        evidence_complete,
+    )
+    metrics = {
+        "reward": float(all(value == 1.0 for value in values)),
+        "answer_correct": answer_correct,
+        "threaded_reply": threaded_reply,
+        "evidence_complete": evidence_complete,
+    }
+    return metrics, {
+        "question_event_id": question_id,
+        "question_channel_id": question_channel,
+        "answer_message_id": answer.get("id") if answer else None,
+        "answer_content": content,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--evidence", type=Path, required=True)
+    parser.add_argument("--reward", type=Path, required=True)
+    parser.add_argument("--details", type=Path, required=True)
+    args = parser.parse_args()
+    try:
+        metrics, details = score_evidence(
+            json.loads(args.evidence.read_text(encoding="utf-8"))
+        )
+    except (OSError, json.JSONDecodeError) as error:
+        metrics, details = _zero(), {"error": str(error)}
+    args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8")
+    args.details.write_text(
+        json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/harbor-buzz-orchestra/README.md

@@ -69,8 +69,8 @@ directory of this harness, not a subdirectory of it — scores Buzz product
 behavior alongside task correctness. It covers direct thread replies, callback
 user mentions, targeted reads of named paths, exact channel membership,
 multiline delivery, non-waking narrative names, batched reports, cross-thread
-isolation, and ambiguous identities. Run one task with the production base
-prompt from the checked-out source build:
+isolation, ambiguous identities, and explicit cold-memory retrieval. Run one
+task with the production base prompt from the checked-out source build:
 
 ```bash
 just benchmark \

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py

@@ -167,6 +167,8 @@ async def run(
                     "--name",
                     credential.agent_id,
                 )
+            await self._seed_memories(orchestrator, trial)
+            for credential in trial.credentials:
                 agents.append(
                     await self._launch_agent(
                         environment=environment,
@@ -786,6 +788,39 @@ async def _verify_m1_output(
                 f"and its stripped text must equal 'Hello, world!' ({detail})"
             )
 
+    async def _seed_memories(
+        self, credential: AgentCredential, trial: TrialHandle
+    ) -> None:
+        """Seed task-declared cold memory without exposing its value to the agent."""
+        for seed in fixture_for(trial.task_name).memory_seeds:
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    self.buzz_cli_binary,
+                    "mem",
+                    "set",
+                    seed.slug,
+                    "-",
+                    stdin=asyncio.subprocess.PIPE,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    env={
+                        **os.environ,
+                        "BUZZ_RELAY_URL": self._user_relay_url(trial),
+                        "BUZZ_PRIVATE_KEY": credential.nostr_secret_key,
+                        "BUZZ_AUTH_TAG": credential.nostr_auth_tag,
+                    },
+                )
+                _, stderr = await process.communicate(seed.value.encode())
+            except OSError as error:
+                raise RuntimeLaunchError(
+                    f"cannot seed cold memory {seed.slug!r}: {error}"
+                ) from None
+            if process.returncode != 0:
+                detail = stderr.decode(errors="replace").strip()
+                raise RuntimeLaunchError(
+                    f"buzz mem set {seed.slug} - exited {process.returncode}: {detail}"
+                )
+
     async def _send(
         self,
         credential: AgentCredential,

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/task_fixtures.py

@@ -32,6 +32,14 @@ class ScriptedMessage:
     mention_orchestrator: bool = True
 
 
+@dataclass(frozen=True, slots=True)
+class MemorySeed:
+    """A cold-memory value seeded under the orchestrator's identity."""
+
+    slug: str
+    value: str
+
+
 @dataclass(frozen=True, slots=True)
 class BuzzTaskFixture:
     """Relay state a task needs before the agent receives its prompt."""
@@ -40,6 +48,7 @@ class BuzzTaskFixture:
     scripted_messages: tuple[ScriptedMessage, ...] = ()
     observe_channel_names: tuple[str, ...] = ()
     user_display_name: str | None = None
+    memory_seeds: tuple[MemorySeed, ...] = ()
     # Whether the task's verifier grades the exported relay snapshot. Only
     # these tasks fail when the export fails; a Terminal-Bench task is graded
     # by its own tests and must not be errored by a snapshot hiccup.
@@ -59,6 +68,7 @@ class BuzzTaskFixture:
 INTERLEAVED_AGENT_REPORTS_TASK = "interleaved-agent-reports"
 CROSS_THREAD_REQUESTS_TASK = "cross-thread-requests"
 AMBIGUOUS_USER_MENTION_TASK = "ambiguous-user-mention"
+MEMORY_RETRIEVAL_TASK = "memory-retrieval"
 
 _CREATE_CHANNEL_FIXTURE = BuzzTaskFixture(
     directory=tuple(
@@ -161,6 +171,20 @@ class BuzzTaskFixture:
     requires_evidence=True,
 )
 
+_MEMORY_RETRIEVAL_FIXTURE = BuzzTaskFixture(
+    user_display_name="Amelia Rose Bennett",
+    memory_seeds=(
+        MemorySeed(
+            slug="finance-conventions-7f2a",
+            value=(
+                "For GPV calculation, always use the net_gpv column, not the "
+                "gross_gpv column."
+            ),
+        ),
+    ),
+    requires_evidence=True,
+)
+
 _FIXTURES = {
     CREATE_CHANNEL_TASK: _CREATE_CHANNEL_FIXTURE,
     USER_MENTION_TASK: _USER_MENTION_FIXTURE,
@@ -173,6 +197,7 @@ class BuzzTaskFixture:
     INTERLEAVED_AGENT_REPORTS_TASK: _INTERLEAVED_AGENT_REPORTS_FIXTURE,
     CROSS_THREAD_REQUESTS_TASK: _CROSS_THREAD_REQUESTS_FIXTURE,
     AMBIGUOUS_USER_MENTION_TASK: _AMBIGUOUS_USER_MENTION_FIXTURE,
+    MEMORY_RETRIEVAL_TASK: _MEMORY_RETRIEVAL_FIXTURE,
 }
 
 

benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md

@@ -9,6 +9,13 @@ endpoint string remains the join key.
 Every key in these files must be a manifest endpoint name; the loader treats
 all entries as endpoint configs (no comment keys).
 
+## openai-live-wire-debug.json
+
+Diagnostic variant of `openai-live.json` for local runs. It enables
+`acp::wire=debug`, so retained agent stdout logs include full ACP messages,
+including tool-call arguments and results. These logs may contain prompt or
+command content; keep them local. The verifier and reward do not read them.
+
 ## m1-local.json
 
 M1 wiring proof: both placeholder endpoints resolve to one local llama-server

benchmarks/harbor-buzz-orchestra/testbed/endpoints/openai-live-wire-debug.json

@@ -0,0 +1,9 @@
+{
+  "gpt-5.6-luna": {
+    "provider": "openai",
+    "api_key_env": "OPENAI_COMPAT_API_KEY",
+    "env": {
+      "RUST_LOG": "acp::wire=debug"
+    }
+  }
+}

benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py

@@ -77,6 +77,7 @@ def test_buzz_task_metadata_defines_the_expected_layers():
             "user-mention",
             "read-named-path-outside-workspace",
             "multiline-message",
+            "memory-retrieval",
             "narrative-agent-names",
         },
         "workflow": {
@@ -167,7 +168,7 @@ def test_explicit_attempts_override_keeps_one_mixed_buzz_job():
     (run,) = benchmark.plan_benchmark_runs(args)
 
     assert run.attempts == 7
-    assert len(run.include_task) == 9
+    assert len(run.include_task) == 10
 
     layered = benchmark.parse_args(
         [

benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py

@@ -1,5 +1,6 @@
 """The container runtime must launch the production stack, unmodified."""
 
+import asyncio
 import hashlib
 import json
 import re
@@ -343,6 +344,57 @@ async def test_launch_wires_the_desktop_environment(tmp_path, configured, expect
     )
 
 
+def test_memory_task_disables_auto_memory_injection(tmp_path):
+    manifest = write_manifest(tmp_path)
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    trial = replace(trial_handle((orch,)), task_name="memory-retrieval")
+
+    env = runtime(tmp_path)._agent_env(
+        trial=trial,
+        credential=orch,
+        agent_class=manifest.roster[0],
+        endpoint=EndpointLaunchConfig("anthropic", "ANTHROPIC_API_KEY"),
+        remote_prompt="/prompt.md",
+    )
+
+    assert env["BUZZ_ACP_CHANNELS"] == "channel"
+    assert env["BUZZ_ACP_NO_MEMORY"] == "true"
+
+
+@pytest.mark.asyncio
+async def test_memory_seed_uses_agent_credentials_and_stdin(tmp_path, monkeypatch):
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    trial = replace(trial_handle((orch,)), task_name="memory-retrieval")
+    captured = {}
+
+    class Process:
+        returncode = 0
+
+        async def communicate(self, value):
+            captured["value"] = value
+            return b"", b"wrote finance-conventions-7f2a"
+
+    async def create_subprocess_exec(*args, **kwargs):
+        captured["args"] = args
+        captured["env"] = kwargs["env"]
+        return Process()
+
+    monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess_exec)
+
+    await runtime(tmp_path)._seed_memories(orch, trial)
+
+    assert captured["args"][1:] == (
+        "mem",
+        "set",
+        "finance-conventions-7f2a",
+        "-",
+    )
+    assert captured["env"]["BUZZ_PRIVATE_KEY"] == orch.nostr_secret_key
+    assert captured["value"] == (
+        b"For GPV calculation, always use the net_gpv column, not the gross_gpv column."
+    )
+
+
 def test_runtime_validates_construction_bounds(tmp_path):
     # 0 is legal and means unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial
     # budget is the clock. Only negatives are rejected.