Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions plugins/nemo-evaluator/src/nemo_evaluator/intake/mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@

from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore
from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial
from nemo_platform.types.intake.evaluation_context_param import EvaluationContextParam
from nemo_platform.types.intake.evaluator_result_create_params import EvaluatorResultCreateParams
from nemo_platform.types.intake.evaluator_result_data_type import EvaluatorResultDataType
from nemo_platform.types.intake.experiment_context_param import ExperimentContextParam
from nemo_platform.types.intake.ingest.atif_agent_param import AtifAgentParam
from nemo_platform.types.intake.ingest.atif_create_params import AtifCreateParams
from nemo_platform.types.intake.ingest.atif_final_metrics_param import AtifFinalMetricsParam
Expand Down Expand Up @@ -69,14 +69,14 @@ def session_id_for(run_id: str, trial_id: str) -> str:
return f"{run_id}:{trial_id}"


def run_task_to_experiment_context(trial: AgentEvalTrial, *, experiment_id: str) -> ExperimentContextParam:
"""Build the lean ingest ``experiment_context`` for a trial.
def run_task_to_evaluation_context(trial: AgentEvalTrial, *, experiment_id: str) -> EvaluationContextParam:
"""Build the lean ingest ``evaluation_context`` for a trial.

Only ``experiment_id`` and ``test_case_id`` live here. Dataset, group, and
free-form metadata belong on the Experiment entity (created separately via
the platform Experiments SDK), not on the per-ingest context.
Only ``evaluation_id`` (the Evaluation's name — ``experiment_id`` holds it) and
``test_case_id`` live here. Dataset, group, and free-form metadata belong on the
Evaluation entity (created separately via the platform SDK), not on the per-ingest context.
"""
return {"experiment_id": experiment_id, "test_case_id": trial.task_id}
return {"evaluation_id": experiment_id, "test_case_id": trial.task_id}


def trial_to_atif_ingest(
Expand Down Expand Up @@ -107,7 +107,7 @@ def trial_to_atif_ingest(
"session_id": session_id_for(run_id, trial.id),
"agent": agent,
"steps": [step],
"experiment_context": run_task_to_experiment_context(trial, experiment_id=experiment_id),
"evaluation_context": run_task_to_evaluation_context(trial, experiment_id=experiment_id),
}
if final_metrics is not None:
body["final_metrics"] = final_metrics
Expand Down
12 changes: 6 additions & 6 deletions plugins/nemo-evaluator/tests/intake/test_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from nemo_evaluator.intake.mapping import (
ATIF_SCHEMA_VERSION,
DEFAULT_AGENT_VERSION,
run_task_to_experiment_context,
run_task_to_evaluation_context,
score_to_evaluator_results,
session_id_for,
trial_to_atif_ingest,
Expand Down Expand Up @@ -72,12 +72,12 @@ def test_session_id_is_stable_per_trial() -> None:
assert session_id_for("run-1", "trial-1") == "run-1:trial-1"


# --- run_task_to_experiment_context -----------------------------------------
# --- run_task_to_evaluation_context -----------------------------------------


def test_experiment_context_is_lean() -> None:
context = run_task_to_experiment_context(_trial(task_id="task-42"), experiment_id="bench-x-variant")
assert context == {"experiment_id": "bench-x-variant", "test_case_id": "task-42"}
def test_evaluation_context_is_lean() -> None:
context = run_task_to_evaluation_context(_trial(task_id="task-42"), experiment_id="bench-x-variant")
assert context == {"evaluation_id": "bench-x-variant", "test_case_id": "task-42"}


# --- trial_to_atif_ingest ---------------------------------------------------
Expand All @@ -95,7 +95,7 @@ def test_trial_to_atif_ingest_shape() -> None:
assert body["session_id"] == "run-1:t-1"
assert body["agent"] == {"name": "my-agent", "version": DEFAULT_AGENT_VERSION, "model_name": "gpt-4o"}
assert body["steps"] == [{"source": "agent", "step_id": 1, "message": "final answer"}]
assert body["experiment_context"] == {"experiment_id": "exp-1", "test_case_id": "task-1"}
assert body["evaluation_context"] == {"evaluation_id": "exp-1", "test_case_id": "task-1"}
assert "final_metrics" not in body


Expand Down
2 changes: 1 addition & 1 deletion plugins/nemo-evaluator/tests/intake/test_publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ async def test_publishes_trajectory_and_scores() -> None:

assert len(client.atif_calls) == 1
assert client.atif_calls[0]["session_id"] == "run-1:t-1"
assert client.atif_calls[0]["experiment_context"] == {"experiment_id": "exp-1", "test_case_id": "task-1"}
assert client.atif_calls[0]["evaluation_context"] == {"evaluation_id": "exp-1", "test_case_id": "task-1"}
# 3 metric outputs across the two score records -> 3 evaluator-result rows.
assert len(client.eval_calls) == 3
assert {call["name"] for call in client.eval_calls} == {"accuracy.score", "accuracy.passed", "latency.p50"}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,9 @@ async def test_publish_to_intake_round_trip(platform_base_url: str) -> None:
trace = traces[0]
assert trace.session_id == t1.session_id
assert trace.root_span_id == t1.span_id
assert trace.experiment_context is not None
assert trace.experiment_context.experiment_id == EXPERIMENT_NAME
assert trace.experiment_context.test_case_id == "task-1"
assert trace.evaluation_context is not None
assert trace.evaluation_context.evaluation_id == EXPERIMENT_NAME
assert trace.evaluation_context.test_case_id == "task-1"

# --- trial-1 scores: every field, every data_type coercion.
rows = await client.intake.spans.evaluator_results.list(t1.span_id, workspace=WORKSPACE)
Expand Down