feat(runs): attach eval runs to an evaluation a launcher already created

packages/prime-runs/src/prime_runs/backend.py

@@ -20,6 +20,10 @@
 class Backend(Protocol):
     def create(self, spec: RunSpec) -> RunHandle: ...
 
+    def attach(self, run_id: str) -> RunHandle:
+        """A handle for a run a launcher already created; no request."""
+        ...
+
     def update(
         self,
         run_id: str,
@@ -49,7 +53,9 @@ class EvalsBackend:
     """Environments are resolved through the hub's get-or-create, so a local run
     uploads without ``prime env push``. The API has no producer-facing way to mark
     a run failed, so a non-completed terminal state is recorded in
-    ``metadata.prime_runs`` and the run keeps showing as running."""
+    ``metadata.prime_runs`` and the run keeps showing as running. A run a launcher
+    already created (a hosted evaluation) is attached to instead (:meth:`attach`):
+    the launcher owns its status, this process streams into it and completes it."""
 
     def __init__(
         self,
@@ -95,6 +101,15 @@ def create(self, spec: RunSpec) -> RunHandle:
             url=response.get("viewer_url") or self.url_for(run_id),
         )
 
+    def attach(self, run_id: str) -> RunHandle:
+        """A handle for an evaluation a launcher already created — a hosted eval's
+        sandbox is handed its id as ``$EVALUATION_ID``. No request: the first write
+        proves access. The launcher created the run with its environments and
+        config, so nothing is resolved or registered here; on a clean finish the
+        run is finalized like one this process created, while a failure is left
+        for the launcher to mark from the process exit."""
+        return RunHandle(id=run_id, url=self.url_for(run_id))
+
     def url_for(self, run_id: str) -> str:
         return f"{self._frontend_url}/dashboard/evaluations/{run_id}"
 
@@ -211,6 +226,9 @@ class DisabledBackend:
     def create(self, spec: RunSpec) -> RunHandle:
         return RunHandle(id=disabled_run_id())
 
+    def attach(self, run_id: str) -> RunHandle:
+        return RunHandle(id=run_id)
+
     def update(self, run_id: str, **kwargs: Any) -> None:
         return None
 

packages/prime-runs/src/prime_runs/run.py

@@ -269,11 +269,12 @@ def _finish_once(
         self._metrics_worker.close(timeout=time_left(deadline))
 
         if self._owns_lifecycle:
+            # An attached run's config document is the launcher's: it created the
+            # run from its own config and reads it back, so only the summary goes up.
+            config = None if self._attached else (self.config or None)
             self._teardown_step(
                 "updating the run",
-                lambda: self._backend.update(
-                    self.id, config=self.config or None, summary=self.summary or None
-                ),
+                lambda: self._backend.update(self.id, config=config, summary=self.summary or None),
             )
             if self._attached and resolved is not RunStatus.COMPLETED:
                 # The launcher marks its own run failed; reporting it here would race that.
@@ -421,12 +422,16 @@ def init(
 
     ``kind="train"`` opens an external training run: ``model`` is the base
     model, ``environments`` the hub ids, ``training`` the display fields, and a
-    team is required. ``id`` attaches to an external run a launcher already
-    created (``$RUN_ID``): nothing is registered, the platform keeps the run's
-    failure marking, and a clean finish still completes it. ``base_url`` is the
-    platform origin; a hosted run passes the internal RFT root its launcher
-    injects (``$PRIME_API_BASE``, ``…/api/internal/rft``), which serves attached
-    runs only.
+    team is required.
+
+    ``id`` attaches to a run a launcher already created instead of creating one:
+    a hosted training run's ``$RUN_ID``, or a hosted evaluation's
+    ``$EVALUATION_ID``. Nothing is registered, ``environments`` and ``config``
+    are the launcher's to record, the platform keeps the run's failure marking,
+    and a clean finish still completes it. ``base_url`` is the platform origin;
+    a hosted training run passes the internal RFT root its launcher injects
+    (``$PRIME_API_BASE``, ``…/api/internal/rft``), which serves attached runs
+    only; a hosted evaluation uses the public API with its sandbox key.
     """
     settings = Config()
     api_key = api_key if api_key is not None else settings.api_key
@@ -435,10 +440,6 @@ def init(
 
     if kind not in ("eval", "train"):
         raise ConfigurationError(f"kind={kind!r} is not one of 'eval' or 'train'")
-    if id is not None and kind != "train":
-        raise ConfigurationError(
-            "id= attaches to an existing training run; an eval run is always created here."
-        )
     if training is not None and kind != "train":
         raise ConfigurationError("training= only applies to kind='train'")
 
@@ -472,13 +473,12 @@ def init(
             )
         client = PlatformClient(api_key=api_key, base_url=base_url, timeout=DEFAULT_TIMEOUT)
         if kind == "train":
-            rft = RftBackend(client, frontend_url=settings.frontend_url, team_id=team_id)
-            backend = rft
+            backend = RftBackend(client, frontend_url=settings.frontend_url, team_id=team_id)
         else:
             backend = EvalsBackend(client, frontend_url=settings.frontend_url, team_id=team_id)
         try:
-            if kind == "train" and id is not None:
-                handle = rft.attach(id)
+            if id is not None:
+                handle = backend.attach(id)
                 attached = True
             else:
                 handle = backend.create(spec)

packages/prime-runs/tests/test_evals_backend.py

@@ -42,6 +42,16 @@ def test_create_resolves_environment_names_through_the_hub(make_platform_client,
     assert handle.url == "https://app.example/dashboard/evaluations/eval-abc"
 
 
+def test_attach_makes_no_request(make_platform_client, eval_routes):
+    backend, handler = make_backend(make_platform_client, eval_routes)
+
+    handle = backend.attach("eval-hosted")
+
+    assert handle.id == "eval-hosted"
+    assert handle.url == "https://app.example/dashboard/evaluations/eval-hosted"
+    assert handler.paths() == []
+
+
 def test_an_explicit_environment_id_skips_the_hub(make_platform_client, eval_routes):
     backend, handler = make_backend(make_platform_client, eval_routes)
 

packages/prime-runs/tests/test_init.py

@@ -69,14 +69,15 @@ def _init(routes=None, **kwargs):
             "prime_runs.run.PlatformClient", lambda **_: make_platform_client(handler)
         )
         monkeypatch.setattr("prime_runs.run.TracesSink", lambda **_: _NullSink())
-        run = pr.init(
-            name="test-run",
-            environments=["gsm8k"],
-            model="Qwen3-8B",
-            framework="verifiers",
-            api_key="test-key",
+        params = {
+            "name": "test-run",
+            "environments": ["gsm8k"],
+            "model": "Qwen3-8B",
+            "framework": "verifiers",
+            "api_key": "test-key",
             **kwargs,
-        )
+        }
+        run = pr.init(**params)
         return run, handler
 
     return _init
@@ -192,6 +193,85 @@ def test_the_end_to_end_shape_a_producer_writes(online):
     assert run.errors == []
 
 
+# ------------------------------------------------------------------ attach
+
+
+@pytest.fixture
+def hosted_eval_routes(eval_routes):
+    """A launcher created ``eval-hosted``; the sandbox may write to it but never creates."""
+
+    def refuse_create(request):
+        import httpx
+
+        return httpx.Response(500, json={"detail": "an attached run must not create"})
+
+    return {
+        "POST /api/v1/evaluations/": refuse_create,
+        "POST /api/v1/evaluations/eval-hosted/samples": {"samples_pushed": 1},
+        "POST /api/v1/evaluations/eval-hosted/finalize": {"status": "PROCESSING"},
+        "PUT /api/v1/evaluations/eval-hosted": {"status": "RUNNING"},
+    }
+
+
+def test_an_eval_run_attaches_to_a_launcher_created_evaluation(online, hosted_eval_routes):
+    run, handler = online(routes=hosted_eval_routes, id="eval-hosted")
+
+    assert run.id == "eval-hosted"
+    assert run.attached is True
+    # No create response to take a viewer URL from: built from the dashboard origin.
+    assert run.url is not None and run.url.endswith("/dashboard/evaluations/eval-hosted")
+    # Neither the hub nor the evaluations API was asked to create anything.
+    assert handler.paths() == []
+
+    run.log_traces([make_episode("ep-1", [make_trace()])])
+    run.finish(summary={"avg_reward": 1.0})
+
+    posted = handler.bodies_for("/api/v1/evaluations/eval-hosted/samples")
+    assert [s["sample_id"] for body in posted for s in body["samples"]] == ["ep-1"]
+    # The launcher's config document is left alone: only the summary goes up.
+    assert handler.bodies_for("/api/v1/evaluations/eval-hosted") == [
+        {"metrics": {"avg_reward": 1.0}}
+    ]
+    # A clean finish still completes the launcher's run.
+    assert handler.bodies_for("/api/v1/evaluations/eval-hosted/finalize") == [
+        {"metrics": {"avg_reward": 1.0}}
+    ]
+    assert run.status is RunStatus.COMPLETED
+    assert run.errors == []
+
+
+def test_an_attached_eval_run_leaves_failure_marking_to_the_launcher(
+    online, hosted_eval_routes, caplog
+):
+    run, handler = online(routes=hosted_eval_routes, id="eval-hosted")
+
+    with caplog.at_level("INFO"):
+        run.fail("boom")
+
+    assert "POST /api/v1/evaluations/eval-hosted/finalize" not in handler.paths()
+    # No metadata.prime_runs failure marker either: the launcher reads the exit code.
+    assert handler.bodies_for("/api/v1/evaluations/eval-hosted") == []
+    assert "leaving it to the launcher" in caplog.text
+    assert run.status is RunStatus.FAILED
+
+
+def test_an_attached_eval_run_needs_no_environments(online, hosted_eval_routes):
+    run, handler = online(routes=hosted_eval_routes, id="eval-hosted", environments=None)
+
+    assert run.id == "eval-hosted"
+    assert handler.paths() == []
+    run.finish()
+
+
+def test_a_disabled_eval_run_keeps_an_attached_id():
+    run = pr.init(mode="disabled", id="eval-hosted")
+
+    assert run.id == "eval-hosted"
+    assert run.attached is False  # nothing to hand back to
+    run.finish()
+    assert run.status is RunStatus.COMPLETED
+
+
 # --------------------------------------------------------------------- fork
 
 
@@ -412,11 +492,10 @@ def test_a_disabled_training_run_keeps_an_attached_id(tmp_path):
 @pytest.mark.parametrize(
     "kwargs",
     [
-        {"kind": "eval", "id": "eval-1"},
         {"kind": "eval", "training": pr.TrainingSpec()},
         {"kind": "serve"},
     ],
-    ids=["eval-with-id", "eval-with-training", "unknown-kind"],
+    ids=["eval-with-training", "unknown-kind"],
 )
 def test_kind_arguments_are_checked_before_anything_else(kwargs):
     with pytest.raises(ConfigurationError):