From 92467c0d2f485eb868f98e65cfe53aceecbbf43d Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:35:43 -0600 Subject: [PATCH] refactor(evaluator): rename experiment_id to evaluation_name in publish-to-intake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `publish_to_intake` and the mapping helpers took an `experiment_id` argument that was never an id and never an Experiment: Intake resolves the entity by name, and after the Experiment -> Evaluation rename the thing being named is an Evaluation. `client.experiments` now means Experiment *groups*, so the old name pointed at the wrong entity as well as the wrong field. Passing a real entity id gets a 400 ("Evaluation 'X' must be created before it can be logged"). Renames the argument on `publish_to_intake`, `trial_to_atif_ingest`, and `run_task_to_evaluation_context`, and the `PublishReport.experiment_id` field, to `evaluation_name`, and updates the docstrings to say which entity has to exist first. The wire field stays `evaluation_id` — that is Intake's contract, and the docstring now calls out that it carries a name despite the suffix. No behavior change. Safe as a plain rename because the module has no production callers yet; only its own tests use it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- .../src/nemo_evaluator/intake/mapping.py | 16 +++++++------- .../src/nemo_evaluator/intake/publish.py | 21 ++++++++++--------- .../tests/intake/test_mapping.py | 10 ++++----- .../tests/intake/test_publish.py | 18 ++++++++-------- .../integration/test_publish_to_intake.py | 18 ++++++++-------- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py index 5f8de82d5f..f6f8eca199 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py @@ -69,21 +69,23 @@ def session_id_for(run_id: str, trial_id: str) -> str: return f"{run_id}:{trial_id}" -def run_task_to_evaluation_context(trial: AgentEvalTrial, *, experiment_id: str) -> EvaluationContextParam: +def run_task_to_evaluation_context(trial: AgentEvalTrial, *, evaluation_name: str) -> EvaluationContextParam: """Build the lean ingest ``evaluation_context`` for a trial. - Only ``evaluation_id`` (the Evaluation's name — ``experiment_id`` holds it) and - ``test_case_id`` live here. Dataset, group, and free-form metadata belong on the - Evaluation entity (created separately via the platform SDK), not on the per-ingest context. + Only ``evaluation_id`` and ``test_case_id`` live here. Despite the ``_id`` suffix, + the wire field ``evaluation_id`` carries the Evaluation's *name* — that is what Intake + resolves the entity by — so ``evaluation_name`` maps straight onto it. Dataset, group, + and free-form metadata belong on the Evaluation entity (created separately via the + platform SDK), not on the per-ingest context. """ - return {"evaluation_id": experiment_id, "test_case_id": trial.task_id} + return {"evaluation_id": evaluation_name, "test_case_id": trial.task_id} def trial_to_atif_ingest( trial: AgentEvalTrial, *, run_id: str, - experiment_id: str, + evaluation_name: str, agent_name: str, agent_version: str = DEFAULT_AGENT_VERSION, model_name: str | None = None, @@ -107,7 +109,7 @@ def trial_to_atif_ingest( "session_id": session_id_for(run_id, trial.id), "agent": agent, "steps": [step], - "evaluation_context": run_task_to_evaluation_context(trial, experiment_id=experiment_id), + "evaluation_context": run_task_to_evaluation_context(trial, evaluation_name=evaluation_name), } if final_metrics is not None: body["final_metrics"] = final_metrics diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py index 11a2acf05b..7f7074dbfc 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/intake/publish.py @@ -8,8 +8,8 @@ there is no feature flag — optionality is structural: you make the call or you don't, and the platform client is a required argument. -It references an **existing** Experiment (created by the caller via the platform -Experiments SDK) and never creates one. Per Trial it: POSTs the ATIF trajectory, +It references an **existing** Evaluation (created by the caller via the platform +Evaluations SDK) and never creates one. Per Trial it: POSTs the ATIF trajectory, resolves the trajectory's root span, then POSTs one evaluator-result per metric output. All request shapes come from :mod:`nemo_evaluator.intake.mapping`; the HTTP calls go through the generated platform SDK's ``intake`` resources. @@ -71,7 +71,7 @@ class PublishReport(BaseModel): model_config = ConfigDict(extra="forbid") - experiment_id: str = Field(description="Experiment the results were published under.") + evaluation_name: str = Field(description="Name of the Evaluation the results were published under.") workspace: str = Field(description="Workspace the writes targeted.") run_id: str = Field(description="Source AgentEvalResult run id.") published_trials: list[PublishedTrial] = Field( @@ -97,14 +97,14 @@ async def publish_to_intake( result: AgentEvalResult, *, platform: AsyncNeMoPlatform, - experiment_id: str, + evaluation_name: str, workspace: str | None = None, agent_name: str = "agent", agent_version: str = mapping.DEFAULT_AGENT_VERSION, model_name: str | None = None, max_concurrency: int = DEFAULT_MAX_CONCURRENCY, ) -> PublishReport: - """Publish a completed ``AgentEvalResult`` to Intake under an existing Experiment. + """Publish a completed ``AgentEvalResult`` to Intake under an existing Evaluation. For each trial: POST the ATIF trajectory, resolve its root span, then POST one evaluator-result per metric output. Trials are published concurrently up to @@ -117,9 +117,10 @@ async def publish_to_intake( touched, so the caller can re-run ``publish_to_intake`` once the issue is fixed to publish the remaining trials. (Re-publish is not yet idempotent — see ask X1.) - ``experiment_id`` must reference an Experiment that already exists — ATIF ingest - rejects unknown experiments with HTTP 400. Creating the Experiment/group is a - separate, caller-side step via the platform Experiments SDK. + ``evaluation_name`` must name an Evaluation that already exists in ``workspace`` — + Intake resolves the Evaluation entity by name and ATIF ingest rejects an unknown or + deleted one with HTTP 400. Creating the Evaluation (and the Experiment group it + belongs to) is a separate, caller-side step via the platform SDK. Agent identity (``agent_name``/``agent_version``/``model_name``) is taken as arguments because it lives on the run *target*, which ``AgentEvalResult`` does @@ -139,7 +140,7 @@ async def _publish_trial(trial: AgentEvalTrial) -> PublishedTrial: body = mapping.trial_to_atif_ingest( trial, run_id=result.run_id, - experiment_id=experiment_id, + evaluation_name=evaluation_name, agent_name=agent_name, agent_version=agent_version, model_name=model_name, @@ -177,7 +178,7 @@ async def _publish_trial(trial: AgentEvalTrial) -> PublishedTrial: failures.append((trial.id, outcome)) report = PublishReport( - experiment_id=experiment_id, + evaluation_name=evaluation_name, workspace=resolved_workspace, run_id=result.run_id, published_trials=published, diff --git a/plugins/nemo-evaluator/tests/intake/test_mapping.py b/plugins/nemo-evaluator/tests/intake/test_mapping.py index 4e5cb70e19..1a3769a76f 100644 --- a/plugins/nemo-evaluator/tests/intake/test_mapping.py +++ b/plugins/nemo-evaluator/tests/intake/test_mapping.py @@ -76,7 +76,7 @@ def test_session_id_is_stable_per_trial() -> None: def test_evaluation_context_is_lean() -> None: - context = run_task_to_evaluation_context(_trial(task_id="task-42"), experiment_id="bench-x-variant") + context = run_task_to_evaluation_context(_trial(task_id="task-42"), evaluation_name="bench-x-variant") assert context == {"evaluation_id": "bench-x-variant", "test_case_id": "task-42"} @@ -87,7 +87,7 @@ def test_trial_to_atif_ingest_shape() -> None: body = trial_to_atif_ingest( _trial(trial_id="t-1", task_id="task-1", output_text="final answer"), run_id="run-1", - experiment_id="exp-1", + evaluation_name="exp-1", agent_name="my-agent", model_name="gpt-4o", ) @@ -100,13 +100,13 @@ def test_trial_to_atif_ingest_shape() -> None: def test_trial_to_atif_ingest_defaults_version_and_omits_model_name() -> None: - body = trial_to_atif_ingest(_trial(), run_id="run-1", experiment_id="exp-1", agent_name="a") + body = trial_to_atif_ingest(_trial(), run_id="run-1", evaluation_name="exp-1", agent_name="a") assert body["agent"] == {"name": "a", "version": "unknown"} assert "model_name" not in body["agent"] def test_trial_to_atif_ingest_handles_missing_output() -> None: - body = trial_to_atif_ingest(_trial(output_text=None), run_id="run-1", experiment_id="exp-1", agent_name="a") + body = trial_to_atif_ingest(_trial(output_text=None), run_id="run-1", evaluation_name="exp-1", agent_name="a") assert body["steps"] == [{"source": "agent", "step_id": 1, "message": ""}] @@ -114,7 +114,7 @@ def test_trial_to_atif_ingest_includes_final_metrics_when_given() -> None: body = trial_to_atif_ingest( _trial(), run_id="run-1", - experiment_id="exp-1", + evaluation_name="exp-1", agent_name="a", final_metrics={"total_prompt_tokens": 10}, ) diff --git a/plugins/nemo-evaluator/tests/intake/test_publish.py b/plugins/nemo-evaluator/tests/intake/test_publish.py index bc616359b5..257c00e59a 100644 --- a/plugins/nemo-evaluator/tests/intake/test_publish.py +++ b/plugins/nemo-evaluator/tests/intake/test_publish.py @@ -135,7 +135,7 @@ async def test_publishes_trajectory_and_scores() -> None: ], ) client = _FakeClient() - report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") assert len(client.atif_calls) == 1 assert client.atif_calls[0]["session_id"] == "run-1:t-1" @@ -166,7 +166,7 @@ async def test_multiple_trials_each_get_their_own_session_and_span() -> None: ], ) client = _FakeClient() - report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") assert len(client.atif_calls) == 2 assert report.trial_count == 2 @@ -177,7 +177,7 @@ async def test_multiple_trials_each_get_their_own_session_and_span() -> None: async def test_trial_without_scores_still_ingests_trajectory() -> None: result = _result(trials=[_trial("t-1")], scores=[]) client = _FakeClient() - report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") assert len(client.atif_calls) == 1 assert len(client.eval_calls) == 0 @@ -188,7 +188,7 @@ async def test_explicit_workspace_overrides_client_default() -> None: result = _result(trials=[_trial("t-1")], scores=[]) client = _FakeClient(workspace="default") report = await publish_to_intake( - result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1", workspace="ws-2" + result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1", workspace="ws-2" ) assert report.workspace == "ws-2" assert client.atif_calls[0]["workspace"] == "ws-2" @@ -198,7 +198,7 @@ async def test_missing_workspace_raises() -> None: result = _result(trials=[_trial("t-1")], scores=[]) client = _FakeClient(workspace=None) with pytest.raises(ValueError, match="workspace"): - await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") async def test_unresolvable_span_raises_publish_error() -> None: @@ -208,14 +208,14 @@ async def test_unresolvable_span_raises_publish_error() -> None: ) client = _FakeClient(root_span_id=None) with pytest.raises(PublishError, match="No root span"): - await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") async def test_ingest_failure_propagates() -> None: result = _result(trials=[_trial("t-1")], scores=[]) client = _FakeClient(atif_fail=True) with pytest.raises(RuntimeError, match="atif ingest 400"): - await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") async def test_failed_and_non_finite_scores_are_skipped_and_reported() -> None: @@ -231,7 +231,7 @@ async def test_failed_and_non_finite_scores_are_skipped_and_reported() -> None: ], ) client = _FakeClient() - report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1") + report = await publish_to_intake(result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1") # Only the finite, completed output is sent to Intake. assert {call["name"] for call in client.eval_calls} == {"accuracy.score"} @@ -256,7 +256,7 @@ async def test_one_trial_failure_does_not_block_others_and_is_reported() -> None with pytest.raises(PublishError) as excinfo: await publish_to_intake( - result, platform=cast(AsyncNeMoPlatform, client), experiment_id="exp-1", max_concurrency=1 + result, platform=cast(AsyncNeMoPlatform, client), evaluation_name="exp-1", max_concurrency=1 ) # The healthy trial still published despite the other failing. diff --git a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py index f673ab16f2..8491383a42 100644 --- a/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py +++ b/plugins/nemo-evaluator/tests/integration/test_publish_to_intake.py @@ -42,9 +42,9 @@ BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080") WORKSPACE = "default" GROUP_NAME = "intake-it-group" -EXPERIMENT_NAME = "intake-it-exp" +EVALUATION_NAME = "intake-it-eval" RUN_ID = "intake-it-run" -NAN_EXPERIMENT_NAME = "intake-it-nan-exp" +NAN_EVALUATION_NAME = "intake-it-nan-eval" NAN_RUN_ID = "intake-it-nan-run" CLICKHOUSE_CONTAINER = "nmp-intake-clickhouse" @@ -174,13 +174,13 @@ def _result() -> AgentEvalResult: async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: async with AsyncNeMoPlatform(base_url=platform_base_url, max_retries=2) as client: - # Precondition: the Experiment must exist before ingest. + # Precondition: the Evaluation (and its Experiment group) must exist before ingest. group = await client.experiments.create( workspace=WORKSPACE, name=GROUP_NAME, description="Intake IT", exist_ok=True ) await client.evaluations.create( workspace=WORKSPACE, - name=EXPERIMENT_NAME, + name=EVALUATION_NAME, experiment_ids=[group.id], dataset_name="intake-it-dataset", dataset_version="v1", @@ -190,7 +190,7 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: report = await publish_to_intake( _result(), platform=client, - experiment_id=EXPERIMENT_NAME, + evaluation_name=EVALUATION_NAME, workspace=WORKSPACE, agent_name="intake-it-agent", model_name="intake-it-model", @@ -200,7 +200,7 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: assert report.evaluator_result_count == 5 published = {trial.trial_id: trial for trial in report.published_trials} - # --- trial-1: trajectory + experiment-context propagation, read back via the Intake API. + # --- trial-1: trajectory + evaluation-context propagation, read back via the Intake API. t1 = published["trial-1"] trace_filter: TraceFilterParam = {"session_id": t1.session_id} traces = [trace async for trace in client.intake.traces.list(workspace=WORKSPACE, filter=trace_filter)] @@ -209,7 +209,7 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None: assert trace.session_id == t1.session_id assert trace.root_span_id == t1.span_id assert trace.evaluation_context is not None - assert trace.evaluation_context.evaluation_id == EXPERIMENT_NAME + assert trace.evaluation_context.evaluation_id == EVALUATION_NAME assert trace.evaluation_context.test_case_id == "task-1" # --- trial-1 scores: every field, every data_type coercion. @@ -277,7 +277,7 @@ async def test_publish_skips_nan_and_failed_scores(platform_base_url: str) -> No group = await client.experiments.create(workspace=WORKSPACE, name=GROUP_NAME, exist_ok=True) await client.evaluations.create( workspace=WORKSPACE, - name=NAN_EXPERIMENT_NAME, + name=NAN_EVALUATION_NAME, experiment_ids=[group.id], dataset_name="intake-it-nan-dataset", dataset_version="v1", @@ -287,7 +287,7 @@ async def test_publish_skips_nan_and_failed_scores(platform_base_url: str) -> No report = await publish_to_intake( _nan_result(), platform=client, - experiment_id=NAN_EXPERIMENT_NAME, + evaluation_name=NAN_EVALUATION_NAME, workspace=WORKSPACE, agent_name="intake-it-agent", )