diff --git a/packages/prime-runs/src/prime_runs/backend.py b/packages/prime-runs/src/prime_runs/backend.py index 61338f6ba..7f4c10ea1 100644 --- a/packages/prime-runs/src/prime_runs/backend.py +++ b/packages/prime-runs/src/prime_runs/backend.py @@ -20,6 +20,10 @@ class Backend(Protocol): def create(self, spec: RunSpec) -> RunHandle: ... + def attach(self, run_id: str) -> RunHandle: + """A handle for a run a launcher already created; no request.""" + ... + def update( self, run_id: str, @@ -49,7 +53,9 @@ class EvalsBackend: """Environments are resolved through the hub's get-or-create, so a local run uploads without ``prime env push``. The API has no producer-facing way to mark a run failed, so a non-completed terminal state is recorded in - ``metadata.prime_runs`` and the run keeps showing as running.""" + ``metadata.prime_runs`` and the run keeps showing as running. A run a launcher + already created (a hosted evaluation) is attached to instead (:meth:`attach`): + the launcher owns its status, this process streams into it and completes it.""" def __init__( self, @@ -95,6 +101,15 @@ def create(self, spec: RunSpec) -> RunHandle: url=response.get("viewer_url") or self.url_for(run_id), ) + def attach(self, run_id: str) -> RunHandle: + """A handle for an evaluation a launcher already created — a hosted eval's + sandbox is handed its id as ``$EVALUATION_ID``. No request: the first write + proves access. The launcher created the run with its environments and + config, so nothing is resolved or registered here; on a clean finish the + run is finalized like one this process created, while a failure is left + for the launcher to mark from the process exit.""" + return RunHandle(id=run_id, url=self.url_for(run_id)) + def url_for(self, run_id: str) -> str: return f"{self._frontend_url}/dashboard/evaluations/{run_id}" @@ -211,6 +226,9 @@ class DisabledBackend: def create(self, spec: RunSpec) -> RunHandle: return RunHandle(id=disabled_run_id()) + def attach(self, run_id: str) -> RunHandle: + return RunHandle(id=run_id) + def update(self, run_id: str, **kwargs: Any) -> None: return None diff --git a/packages/prime-runs/src/prime_runs/run.py b/packages/prime-runs/src/prime_runs/run.py index f640242c0..2de3b73c1 100644 --- a/packages/prime-runs/src/prime_runs/run.py +++ b/packages/prime-runs/src/prime_runs/run.py @@ -269,11 +269,12 @@ def _finish_once( self._metrics_worker.close(timeout=time_left(deadline)) if self._owns_lifecycle: + # An attached run's config document is the launcher's: it created the + # run from its own config and reads it back, so only the summary goes up. + config = None if self._attached else (self.config or None) self._teardown_step( "updating the run", - lambda: self._backend.update( - self.id, config=self.config or None, summary=self.summary or None - ), + lambda: self._backend.update(self.id, config=config, summary=self.summary or None), ) if self._attached and resolved is not RunStatus.COMPLETED: # The launcher marks its own run failed; reporting it here would race that. @@ -421,12 +422,16 @@ def init( ``kind="train"`` opens an external training run: ``model`` is the base model, ``environments`` the hub ids, ``training`` the display fields, and a - team is required. ``id`` attaches to an external run a launcher already - created (``$RUN_ID``): nothing is registered, the platform keeps the run's - failure marking, and a clean finish still completes it. ``base_url`` is the - platform origin; a hosted run passes the internal RFT root its launcher - injects (``$PRIME_API_BASE``, ``…/api/internal/rft``), which serves attached - runs only. + team is required. + + ``id`` attaches to a run a launcher already created instead of creating one: + a hosted training run's ``$RUN_ID``, or a hosted evaluation's + ``$EVALUATION_ID``. Nothing is registered, ``environments`` and ``config`` + are the launcher's to record, the platform keeps the run's failure marking, + and a clean finish still completes it. ``base_url`` is the platform origin; + a hosted training run passes the internal RFT root its launcher injects + (``$PRIME_API_BASE``, ``…/api/internal/rft``), which serves attached runs + only; a hosted evaluation uses the public API with its sandbox key. """ settings = Config() api_key = api_key if api_key is not None else settings.api_key @@ -435,10 +440,6 @@ def init( if kind not in ("eval", "train"): raise ConfigurationError(f"kind={kind!r} is not one of 'eval' or 'train'") - if id is not None and kind != "train": - raise ConfigurationError( - "id= attaches to an existing training run; an eval run is always created here." - ) if training is not None and kind != "train": raise ConfigurationError("training= only applies to kind='train'") @@ -472,13 +473,12 @@ def init( ) client = PlatformClient(api_key=api_key, base_url=base_url, timeout=DEFAULT_TIMEOUT) if kind == "train": - rft = RftBackend(client, frontend_url=settings.frontend_url, team_id=team_id) - backend = rft + backend = RftBackend(client, frontend_url=settings.frontend_url, team_id=team_id) else: backend = EvalsBackend(client, frontend_url=settings.frontend_url, team_id=team_id) try: - if kind == "train" and id is not None: - handle = rft.attach(id) + if id is not None: + handle = backend.attach(id) attached = True else: handle = backend.create(spec) diff --git a/packages/prime-runs/tests/test_evals_backend.py b/packages/prime-runs/tests/test_evals_backend.py index b8ccea974..323c834e4 100644 --- a/packages/prime-runs/tests/test_evals_backend.py +++ b/packages/prime-runs/tests/test_evals_backend.py @@ -42,6 +42,16 @@ def test_create_resolves_environment_names_through_the_hub(make_platform_client, assert handle.url == "https://app.example/dashboard/evaluations/eval-abc" +def test_attach_makes_no_request(make_platform_client, eval_routes): + backend, handler = make_backend(make_platform_client, eval_routes) + + handle = backend.attach("eval-hosted") + + assert handle.id == "eval-hosted" + assert handle.url == "https://app.example/dashboard/evaluations/eval-hosted" + assert handler.paths() == [] + + def test_an_explicit_environment_id_skips_the_hub(make_platform_client, eval_routes): backend, handler = make_backend(make_platform_client, eval_routes) diff --git a/packages/prime-runs/tests/test_init.py b/packages/prime-runs/tests/test_init.py index de301ce7d..9ea1b596f 100644 --- a/packages/prime-runs/tests/test_init.py +++ b/packages/prime-runs/tests/test_init.py @@ -69,14 +69,15 @@ def _init(routes=None, **kwargs): "prime_runs.run.PlatformClient", lambda **_: make_platform_client(handler) ) monkeypatch.setattr("prime_runs.run.TracesSink", lambda **_: _NullSink()) - run = pr.init( - name="test-run", - environments=["gsm8k"], - model="Qwen3-8B", - framework="verifiers", - api_key="test-key", + params = { + "name": "test-run", + "environments": ["gsm8k"], + "model": "Qwen3-8B", + "framework": "verifiers", + "api_key": "test-key", **kwargs, - ) + } + run = pr.init(**params) return run, handler return _init @@ -192,6 +193,85 @@ def test_the_end_to_end_shape_a_producer_writes(online): assert run.errors == [] +# ------------------------------------------------------------------ attach + + +@pytest.fixture +def hosted_eval_routes(eval_routes): + """A launcher created ``eval-hosted``; the sandbox may write to it but never creates.""" + + def refuse_create(request): + import httpx + + return httpx.Response(500, json={"detail": "an attached run must not create"}) + + return { + "POST /api/v1/evaluations/": refuse_create, + "POST /api/v1/evaluations/eval-hosted/samples": {"samples_pushed": 1}, + "POST /api/v1/evaluations/eval-hosted/finalize": {"status": "PROCESSING"}, + "PUT /api/v1/evaluations/eval-hosted": {"status": "RUNNING"}, + } + + +def test_an_eval_run_attaches_to_a_launcher_created_evaluation(online, hosted_eval_routes): + run, handler = online(routes=hosted_eval_routes, id="eval-hosted") + + assert run.id == "eval-hosted" + assert run.attached is True + # No create response to take a viewer URL from: built from the dashboard origin. + assert run.url is not None and run.url.endswith("/dashboard/evaluations/eval-hosted") + # Neither the hub nor the evaluations API was asked to create anything. + assert handler.paths() == [] + + run.log_traces([make_episode("ep-1", [make_trace()])]) + run.finish(summary={"avg_reward": 1.0}) + + posted = handler.bodies_for("/api/v1/evaluations/eval-hosted/samples") + assert [s["sample_id"] for body in posted for s in body["samples"]] == ["ep-1"] + # The launcher's config document is left alone: only the summary goes up. + assert handler.bodies_for("/api/v1/evaluations/eval-hosted") == [ + {"metrics": {"avg_reward": 1.0}} + ] + # A clean finish still completes the launcher's run. + assert handler.bodies_for("/api/v1/evaluations/eval-hosted/finalize") == [ + {"metrics": {"avg_reward": 1.0}} + ] + assert run.status is RunStatus.COMPLETED + assert run.errors == [] + + +def test_an_attached_eval_run_leaves_failure_marking_to_the_launcher( + online, hosted_eval_routes, caplog +): + run, handler = online(routes=hosted_eval_routes, id="eval-hosted") + + with caplog.at_level("INFO"): + run.fail("boom") + + assert "POST /api/v1/evaluations/eval-hosted/finalize" not in handler.paths() + # No metadata.prime_runs failure marker either: the launcher reads the exit code. + assert handler.bodies_for("/api/v1/evaluations/eval-hosted") == [] + assert "leaving it to the launcher" in caplog.text + assert run.status is RunStatus.FAILED + + +def test_an_attached_eval_run_needs_no_environments(online, hosted_eval_routes): + run, handler = online(routes=hosted_eval_routes, id="eval-hosted", environments=None) + + assert run.id == "eval-hosted" + assert handler.paths() == [] + run.finish() + + +def test_a_disabled_eval_run_keeps_an_attached_id(): + run = pr.init(mode="disabled", id="eval-hosted") + + assert run.id == "eval-hosted" + assert run.attached is False # nothing to hand back to + run.finish() + assert run.status is RunStatus.COMPLETED + + # --------------------------------------------------------------------- fork @@ -412,11 +492,10 @@ def test_a_disabled_training_run_keeps_an_attached_id(tmp_path): @pytest.mark.parametrize( "kwargs", [ - {"kind": "eval", "id": "eval-1"}, {"kind": "eval", "training": pr.TrainingSpec()}, {"kind": "serve"}, ], - ids=["eval-with-id", "eval-with-training", "unknown-kind"], + ids=["eval-with-training", "unknown-kind"], ) def test_kind_arguments_are_checked_before_anything_else(kwargs): with pytest.raises(ConfigurationError):