From 4fd7b7aed6336c90dcb8622a81ef2267ccf73c15 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 23 Jun 2026 13:08:16 -0300 Subject: [PATCH 1/2] docs(evaluator): fix SDK submit() examples, FilesetRef import, ModelRef, and leaked titles The Evaluator docs drifted from the current plugin SDK contract. This brings the runnable snippets back in line and fixes two broken tutorial titles. - submit(): add the now-required `metric_bundle_packager=CloudpickleMetricBundlePackager()` (and its import) to every runnable `evaluator.submit(...)` example across index, sdk-resources, and the metrics/* and tutorials/* pages. `run()` examples are unchanged (they do not take the packager). - LLM Judge tutorial: import `FilesetRef` from `nemo_evaluator.sdk` instead of the non-existent `nemo_evaluator_sdk.values.FilesetRef`. - model-configuration: document that local `run()` requires an inline `Model` while remote `submit()` also accepts a `ModelRef`. - test_doc_examples.py: replace the stale `/v2/...` REST script with offline contract checks for the import paths and the submit() packager requirement. - Fix two tutorial pages that rendered `@nemo-nb: hide` as the page title (leaked cell marker in frontmatter `title`); set real titles and drop the duplicate body H1, matching every other evaluator page. Verified: `make docs-check` and `make docs-broken-links` pass; ruff/ty clean; the new contract test passes (7/7); both tutorial titles confirmed via the Fern dev-server render. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- docs/evaluator/index.mdx | 8 +- .../evaluator/metrics/agent-configuration.mdx | 2 + docs/evaluator/metrics/agentic.mdx | 14 + docs/evaluator/metrics/job-management.mdx | 2 + docs/evaluator/metrics/llm-as-a-judge.mdx | 2 + docs/evaluator/metrics/manage-metrics.mdx | 2 + .../evaluator/metrics/model-configuration.mdx | 37 ++- docs/evaluator/metrics/rag.mdx | 52 +++- docs/evaluator/metrics/remote.mdx | 4 + docs/evaluator/metrics/results.mdx | 2 + docs/evaluator/metrics/similarity.mdx | 20 +- docs/evaluator/sdk-resources.mdx | 14 +- docs/evaluator/test_doc_examples.py | 276 ++++++++---------- .../define-run-custom-python-metrics.mdx | 4 +- .../tutorials/run-llm-judge-evaluation.mdx | 12 +- 15 files changed, 274 insertions(+), 177 deletions(-) diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index cf4b63c444..b1d8f5f6cc 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -59,6 +59,7 @@ Submit your evaluation to the Evaluator service using the NeMo Platform SDK: ```python from nemo_evaluator.sdk import Evaluator +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_platform import NeMoPlatform @@ -69,7 +70,12 @@ evaluator: Evaluator = sdk.evaluator local_result = evaluator.run(metric=metric, dataset=dataset, config=config) # Production evaluation as a durable platform job -job = evaluator.submit(metric=metric, dataset=dataset, config=config) +job = evaluator.submit( + metric=metric, + dataset=dataset, + config=config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() ``` diff --git a/docs/evaluator/metrics/agent-configuration.mdx b/docs/evaluator/metrics/agent-configuration.mdx index 012bc52d59..ca855031c9 100644 --- a/docs/evaluator/metrics/agent-configuration.mdx +++ b/docs/evaluator/metrics/agent-configuration.mdx @@ -164,6 +164,7 @@ from nemo_evaluator_sdk import Agent, RunConfigOnline from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric(reference="{{item.expected_answer}}") agent = Agent( url="https://my-nat-agent.example.com", @@ -184,6 +185,7 @@ job = evaluator.submit( {"role": "user", "content": "{{item.question}}"}, ], }, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index 269c1caea6..d45e4c0a05 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -228,6 +228,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ToolCallAccuracyMetric() job = evaluator.submit( @@ -250,6 +251,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -428,6 +430,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig, ToolCallingMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ToolCallingMetric(reference="{{item.tool_calls}}") @@ -462,6 +465,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -568,6 +572,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig, Model from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager judge_model = Model( url="https://integrate.api.nvidia.com/v1/chat/completions", @@ -591,6 +596,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -762,6 +768,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig, Model from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager judge_model = Model( url="https://integrate.api.nvidia.com/v1/chat/completions", @@ -791,6 +798,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -917,6 +925,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig, Model from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager judge_model = Model( url="https://integrate.api.nvidia.com/v1/chat/completions", @@ -954,6 +963,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -1013,6 +1023,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfig, Model from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager judge_model = Model( url="https://integrate.api.nvidia.com/v1/chat/completions", @@ -1031,6 +1042,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -1043,6 +1055,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager judge_model = Model( url="https://integrate.api.nvidia.com/v1/chat/completions", @@ -1077,6 +1090,7 @@ job = evaluator.submit( } ] }, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() diff --git a/docs/evaluator/metrics/job-management.mdx b/docs/evaluator/metrics/job-management.mdx index 65fa3fa576..20391c2aef 100644 --- a/docs/evaluator/metrics/job-management.mdx +++ b/docs/evaluator/metrics/job-management.mdx @@ -21,6 +21,7 @@ from nemo_evaluator.sdk import Evaluator from nemo_platform import NeMoPlatform from nemo_evaluator_sdk import RunConfig from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager sdk = NeMoPlatform( base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), @@ -37,6 +38,7 @@ job = evaluator.submit( {"expected": "Berlin", "output": "Munich"}, ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) print("Submitted job:", job.name) diff --git a/docs/evaluator/metrics/llm-as-a-judge.mdx b/docs/evaluator/metrics/llm-as-a-judge.mdx index a22caec4b2..935ec30bd7 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.mdx +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -298,6 +298,7 @@ For production workloads, submit the same metric and dataset as a durable platfo ```python from nemo_evaluator_sdk import RunConfig, JSONScoreParser, Model, RubricScore, LLMJudgeMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = LLMJudgeMetric( model=Model( @@ -346,6 +347,7 @@ job = evaluator.submit( {"input": "What is 2 + 2?", "output": "4"}, ], config=RunConfig(parallelism=8, limit_samples=100), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) print("Submitted job:", job.name) diff --git a/docs/evaluator/metrics/manage-metrics.mdx b/docs/evaluator/metrics/manage-metrics.mdx index 0784c869c5..79d051fa10 100644 --- a/docs/evaluator/metrics/manage-metrics.mdx +++ b/docs/evaluator/metrics/manage-metrics.mdx @@ -98,6 +98,7 @@ For online evaluations, provide a model or agent target and use the online param ```python from nemo_evaluator_sdk import RunConfig, ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") @@ -108,6 +109,7 @@ job = evaluator.submit( {"expected": "Berlin", "output": "Munich"}, ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index 9691698531..e7f250664a 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -183,7 +183,42 @@ Use plain `RunConfig` for offline evaluations where the dataset already contains ## Model References -The plugin SDK examples on this page use inline `Model` objects. If your deployment resolves platform model entities into model endpoint details, perform that lookup before constructing the `Model`, then pass the resulting inline model to the metric or request. +You can supply the evaluation target two ways. Which one is valid depends on whether you run the evaluation locally or submit it as a durable platform job. + +### Inline `Model` (required for `evaluator.run(...)`) + +`evaluator.run(...)` executes in your local Python process, so it needs the resolved endpoint details inline. Always pass an inline `Model` as the `target` (or as a judge/embeddings field on the metric). If your deployment stores platform model entities, resolve the entity into endpoint details before constructing the `Model`: + +```python +from nemo_evaluator_sdk import Model + +model_entity = client.models.retrieve("my-model", workspace="default") +model = Model( + url=client.models.get_model_entity_route_openai_url(model_entity), + name="my-model", + api_key_secret="NVIDIA_API_KEY", +) + +result = evaluator.run(metric=metric, dataset=dataset, target=model) +``` + +### `ModelRef` (supported by `evaluator.submit(...)`) + +Durable remote `evaluator.submit(...)` jobs additionally accept a `ModelRef` target. A `ModelRef` names a platform model entity (`workspace/model-name`) and is resolved by the evaluator backend when the job runs, so you do not have to resolve the endpoint yourself. Use this for platform-managed model routing: + +```python +from nemo_evaluator_sdk import ModelRef +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + +job = evaluator.submit( + metric=metric, + dataset=dataset, + target=ModelRef(root="default/my-model"), + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) +``` + +`ModelRef` is **not** valid for `evaluator.run(...)`; the local runtime cannot resolve a platform entity. Pass an inline `Model` for local runs and either a `Model` or a `ModelRef` for remote submits. See the [Define and Run Custom Python Metrics](/documentation/evaluate-models/tutorials/define-and-run-custom-python-metrics) tutorial for an end-to-end `ModelRef` + `FilesetRef` submit example. diff --git a/docs/evaluator/metrics/rag.mdx b/docs/evaluator/metrics/rag.mdx index d559118574..68bcd58a5f 100644 --- a/docs/evaluator/metrics/rag.mdx +++ b/docs/evaluator/metrics/rag.mdx @@ -61,7 +61,13 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: ```python -job = evaluator.submit(metric=metric, dataset=dataset) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + +job = evaluator.submit( + metric=metric, + dataset=dataset, + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() ``` @@ -220,9 +226,16 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ContextRecallMetric(judge_model=judge_model) -job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit( + metric=metric, + dataset=offline_rows, + config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() ``` @@ -292,9 +305,16 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ContextPrecisionMetric(judge_model=judge_model) -job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit( + metric=metric, + dataset=offline_rows, + config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() ``` @@ -367,6 +387,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ContextRelevanceMetric(judge_model=judge_model) job = evaluator.submit( @@ -378,6 +400,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -432,6 +455,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ContextEntityRecallMetric(judge_model=judge_model) job = evaluator.submit( @@ -443,6 +468,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -507,6 +533,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = FaithfulnessMetric(judge_model=judge_model) job = evaluator.submit( @@ -515,6 +543,7 @@ job = evaluator.submit( config=online_config, target=generation_model, prompt_template=online_prompt_template, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -578,6 +607,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ResponseGroundednessMetric(judge_model=judge_model) job = evaluator.submit( @@ -586,6 +617,7 @@ job = evaluator.submit( config=online_config, target=generation_model, prompt_template=online_prompt_template, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -648,6 +680,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = NoiseSensitivityMetric(judge_model=judge_model) job = evaluator.submit( @@ -664,6 +698,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -742,6 +777,8 @@ for score in result.aggregate_scores.scores: ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + metric = ResponseRelevancyMetric( judge_model=judge_model, embeddings_model=embeddings_model, @@ -754,6 +791,7 @@ job = evaluator.submit( config=online_config, target=generation_model, prompt_template=online_prompt_template, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -877,7 +915,13 @@ judge_model = Model( For durable remote execution, submit the same metric and dataset that you tested locally: ```python -job = evaluator.submit(metric=metric, dataset=dataset) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + +job = evaluator.submit( + metric=metric, + dataset=dataset, + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() artifacts_dir = job.download_artifacts(path="evaluation_artifacts") print(f"Saved artifacts under {artifacts_dir}") diff --git a/docs/evaluator/metrics/remote.mdx b/docs/evaluator/metrics/remote.mdx index 8886217f4d..b14f66bdc1 100644 --- a/docs/evaluator/metrics/remote.mdx +++ b/docs/evaluator/metrics/remote.mdx @@ -155,6 +155,7 @@ For production workloads, submit the same metric and dataset as a durable platfo ```python from nemo_evaluator_sdk import RunConfig, JSONScoreParser, RemoteScore, RemoteMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = RemoteMetric( url="https://my-evaluation-server.test/evaluate", @@ -182,6 +183,7 @@ job = evaluator.submit( {"reference": "2", "output": "2"}, ], config=RunConfig(parallelism=8), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) print("Submitted job:", job.name) @@ -197,6 +199,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import RunConfig, NemoAgentToolkitRemoteMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = NemoAgentToolkitRemoteMetric( url="http://localhost:8001/evaluate_item", @@ -220,6 +223,7 @@ job = evaluator.submit( } ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() diff --git a/docs/evaluator/metrics/results.mdx b/docs/evaluator/metrics/results.mdx index 07a1b333ea..41bc7d82e6 100644 --- a/docs/evaluator/metrics/results.mdx +++ b/docs/evaluator/metrics/results.mdx @@ -47,6 +47,7 @@ result = evaluator.run( ```python from nemo_evaluator_sdk import RunConfig, ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") @@ -57,6 +58,7 @@ job = evaluator.submit( {"expected": "Berlin", "output": "Munich"}, ], config=RunConfig(parallelism=4), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() ``` diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx index 75138076e1..3be35bf736 100644 --- a/docs/evaluator/metrics/similarity.mdx +++ b/docs/evaluator/metrics/similarity.mdx @@ -31,7 +31,13 @@ evaluator: Evaluator = sdk.evaluator # this object is an Evaluator resource Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: ```python -job = evaluator.submit(metric=metric, dataset=dataset) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + +job = evaluator.submit( + metric=metric, + dataset=dataset, + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() ``` @@ -105,6 +111,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import BLEUMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = BLEUMetric( references=["{{item.reference_1}}", "{{item.reference_2}}"], @@ -126,6 +133,7 @@ job = evaluator.submit( "model_output": "Hello world!", }, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -198,6 +206,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric( reference="{{item.correct_answer | lower | trim}}", @@ -211,6 +220,7 @@ job = evaluator.submit( {"correct_answer": "London", "model_answer": "london "}, {"correct_answer": "Berlin", "model_answer": "Munich"}, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -279,6 +289,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import F1Metric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = F1Metric( reference="{{item.reference}}", @@ -294,6 +305,7 @@ job = evaluator.submit( }, {"reference": "a red apple", "answer": "red apple"}, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -370,6 +382,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import NumberCheckMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = NumberCheckMetric( operation=">", @@ -386,6 +399,7 @@ job = evaluator.submit( {"predicted": "0.5"}, {"predicted": "0.1"}, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -457,6 +471,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import ROUGEMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ROUGEMetric( reference="{{item.reference_summary}}", @@ -475,6 +490,7 @@ job = evaluator.submit( "model_summary": "High winds delayed the launch.", }, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() @@ -563,6 +579,7 @@ for score in result.aggregate_scores.scores: ```python from nemo_evaluator_sdk import StringCheckMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = StringCheckMetric( operation="startswith", @@ -578,6 +595,7 @@ job = evaluator.submit( {"output": "Answer: Success"}, {"output": "Error occurred"}, ], + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) job.wait_until_done() result = job.get_result() diff --git a/docs/evaluator/sdk-resources.mdx b/docs/evaluator/sdk-resources.mdx index ca0ee6d708..b1ed7d36e6 100644 --- a/docs/evaluator/sdk-resources.mdx +++ b/docs/evaluator/sdk-resources.mdx @@ -88,6 +88,7 @@ print(result.aggregate_scores) ```python from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") @@ -96,7 +97,11 @@ dataset = [ {"expected": "Berlin", "output": "Munich"}, ] -job = evaluator.submit(metric=metric, dataset=dataset) +job = evaluator.submit( + metric=metric, + dataset=dataset, + metric_bundle_packager=CloudpickleMetricBundlePackager(), +) job.wait_until_done() result = job.get_result() print(result.aggregate_scores) @@ -138,6 +143,7 @@ evaluator: AsyncEvaluator = client.evaluator import asyncio from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") @@ -148,7 +154,11 @@ dataset = [ async def main() -> None: - job = await evaluator.submit(metric=metric, dataset=dataset) + job = await evaluator.submit( + metric=metric, + dataset=dataset, + metric_bundle_packager=CloudpickleMetricBundlePackager(), + ) await job.wait_until_done() result = await job.get_result() print(result.aggregate_scores) diff --git a/docs/evaluator/test_doc_examples.py b/docs/evaluator/test_doc_examples.py index 2a28489f8c..f15222384a 100644 --- a/docs/evaluator/test_doc_examples.py +++ b/docs/evaluator/test_doc_examples.py @@ -2,167 +2,125 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Test script for evaluator documentation examples. - -Run with: uv run python docs/evaluator/test_doc_examples.py - -FINDINGS: -========= - -1. Job Evaluation (async) - WORKS - - Use system metric URNs: "system/trajectory-evaluation" - - Configure judge in metric_params for metrics that require it - - Metric names use HYPHENS (not underscores): trajectory-evaluation - -2. Live Evaluation (sync) - BUG IN SERVICE - - The evaluate endpoint has a validation bug in the current service build - - Returns "Unable to extract tag using discriminator 'type'" for all requests - - Both inline metrics and MetricRef strings fail - - This needs to be fixed in the service code - -3. RAGAS Metrics - Use for tool_call_accuracy, topic_adherence, etc. - - These are available as RAGAS metric types (not system metrics) - - Support both live and job evaluation - -WORKING JOB PATTERNS: -===================== - -Trajectory Evaluation (system metric, judge required): - spec: - metric: "system/trajectory-evaluation" - dataset: - rows: - - question: "..." - generated_answer: "..." - answer: "..." - intermediate_steps: [NAT format] - metric_params: - judge: - model: - url: "https://nim.int.aire.nvidia.com/v1/chat/completions" - name: "meta/llama-3.2-3b-instruct" - trajectory_used_tools: "tool1,tool2" - -RAGAS Tool Call Accuracy (no judge required): - metric: - type: "tool_call_accuracy" - dataset: - rows: - - user_input: [multi-turn conversation with tool_calls] - reference_tool_calls: [expected tool calls] +"""Contract checks for the Evaluator SDK patterns used in these docs. + +The Evaluator docs are written against the ``nemo_evaluator`` plugin SDK +(``evaluator.run(...)`` / ``evaluator.submit(...)``), not the old +``/v2/.../evaluation/metrics/jobs`` REST endpoints. This module validates the +import paths and call contract that every runnable doc snippet relies on, so the +docs cannot silently drift from the SDK again. + +These checks run fully offline: they exercise import locations and the +client-side argument validation in ``Evaluator.submit`` / ``AsyncEvaluator.submit``. +They do not submit jobs and need no running platform or model credentials. + +Run directly: + uv run python docs/evaluator/test_doc_examples.py + +Or under pytest: + uv run pytest docs/evaluator/test_doc_examples.py -v """ -import time - -import httpx - -BASE_URL = "http://localhost:8080" -WORKSPACE = "doc-test-workspace" - - -def setup_workspace(): - """Create workspace if it doesn't exist.""" - with httpx.Client() as client: - try: - response = client.post(f"{BASE_URL}/v2/workspaces", json={"name": WORKSPACE}) - if response.status_code in (200, 201): - print(f"✓ Created workspace: {WORKSPACE}") - elif response.status_code == 409: - print(f"✓ Workspace exists: {WORKSPACE}") - except Exception as e: - print(f"✗ Error: {e}") - - -def test_trajectory_evaluation_job(): - """Test Trajectory Evaluation - Job (judge required)""" - print("\n[TEST] Trajectory Evaluation - Job (system/trajectory-evaluation)") - - payload = { - "spec": { - "dataset": { - "rows": [{ - "question": "What is the weather?", - "generated_answer": "The weather is sunny.", - "answer": "The weather is sunny.", - "intermediate_steps": [ - { - "payload": { - "event_type": "LLM_END", - "name": "test-model", - "data": { - "input": "What is the weather?", - "output": "Action: weather_tool\nAction Input: {}" - } - } - }, - { - "payload": { - "event_type": "TOOL_END", - "name": "weather_tool", - "data": { - "input": "{}", - "output": "sunny" - } - } - } - ] - }] - }, - "metric": "system/trajectory-evaluation", - "params": {"limit_samples": 1}, - "metric_params": { - "judge": { - "model": { - "url": "https://nim.int.aire.nvidia.com/v1/chat/completions", - "name": "meta/llama-3.2-3b-instruct" - } - }, - "trajectory_used_tools": "weather_tool" - } - } - } - - with httpx.Client(timeout=300.0) as client: - response = client.post(f"{BASE_URL}/v2/workspaces/{WORKSPACE}/evaluation/metrics/jobs", json=payload) - if response.status_code not in (200, 201): - print(f" ✗ Create failed: {response.text[:200]}") - return False - - job_name = response.json()["name"] - print(f" Job: {job_name}") - - for _ in range(30): - status = client.get(f"{BASE_URL}/v2/workspaces/{WORKSPACE}/evaluation/metrics/jobs/{job_name}").json().get("status") - print(f" Status: {status}") - if status in ("completed", "error", "cancelled"): - break - time.sleep(4) - - return status == "completed" - - -def main(): - print("=" * 60) - print("Evaluator Documentation Test") - print("=" * 60) - - setup_workspace() - - passed = 0 - failed = 0 - - if test_trajectory_evaluation_job(): - print(" ✓ PASSED") - passed += 1 - else: - print(" ✗ FAILED") - failed += 1 - - print(f"\n{'=' * 60}") - print(f"Results: {passed} passed, {failed} failed") - print("=" * 60) - - raise SystemExit(1 if failed else 0) +from __future__ import annotations + +import inspect + +import pytest +from nemo_evaluator.sdk import Evaluator +from nemo_platform import NeMoPlatform + + +def test_filesetref_imports_from_platform_sdk() -> None: + """Docs import ``FilesetRef`` from ``nemo_evaluator.sdk`` (platform helpers).""" + from nemo_evaluator.sdk import FilesetRef + + assert FilesetRef is not None + + +def test_filesetref_is_not_in_nemo_evaluator_sdk_values() -> None: + """``FilesetRef`` is NOT exported from ``nemo_evaluator_sdk.values``. + + The LLM Judge tutorial previously imported it from the wrong module, which + fails at import time. Guard against that regression. + """ + import nemo_evaluator_sdk.values as values + + assert not hasattr(values, "FilesetRef") + + +def test_modelref_imports_from_context_agnostic_sdk() -> None: + """Docs import ``ModelRef`` from ``nemo_evaluator_sdk`` (value types).""" + from nemo_evaluator_sdk import ModelRef + + assert ModelRef is not None + + +def test_cloudpickle_packager_import_path() -> None: + """Durable-submit docs import the packager from this exact path.""" + from nemo_evaluator.shared.metric_bundles.cloudpickle import ( + CloudpickleMetricBundlePackager, + ) + + assert CloudpickleMetricBundlePackager is not None + + +def _evaluator() -> Evaluator: + """Build an Evaluator resource without contacting any service. + + Client construction and the ``submit`` argument guard are both offline; the + guard runs before any executor/HTTP work. + """ + client = NeMoPlatform(base_url="http://localhost:8080", workspace="default") + return client.evaluator + + +def test_submit_exposes_metric_bundle_packager_but_run_does_not() -> None: + """``submit`` takes ``metric_bundle_packager``; ``run`` (local, in-process) does not.""" + from nemo_evaluator.sdk import Evaluator + + submit_params = inspect.signature(Evaluator.submit).parameters + run_params = inspect.signature(Evaluator.run).parameters + assert "metric_bundle_packager" in submit_params + assert "metric_bundle_packager" not in run_params + + +def test_submit_requires_metric_bundle_packager() -> None: + """``submit()`` without a packager raises the documented ValueError, offline.""" + from nemo_evaluator_sdk import ExactMatchMetric + + evaluator = _evaluator() + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + dataset = [{"expected": "Paris", "output": "Paris"}] + + with pytest.raises(ValueError, match="metric_bundle_packager is required"): + evaluator.submit(metric=metric, dataset=dataset) + + +def test_run_does_not_require_metric_bundle_packager() -> None: + """``run()`` must not impose the submit-only packager requirement. + + ``run`` executes in-process; reaching the executor (which then needs a live + service) proves the packager guard did not fire. We only assert the failure + is NOT the packager ValueError. + """ + from nemo_evaluator_sdk import ExactMatchMetric + + evaluator = _evaluator() + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + dataset = [{"expected": "Paris", "output": "Paris"}] + + try: + evaluator.run(metric=metric, dataset=dataset) + except ValueError as error: # pragma: no cover - defensive + assert "metric_bundle_packager is required" not in str(error) + except Exception: + # Any non-ValueError (e.g. connection error to the local runtime) is fine; + # it means we got past argument validation. + pass + + +def main() -> None: + raise SystemExit(pytest.main([__file__, "-v"])) if __name__ == "__main__": diff --git a/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx b/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx index 17f888b1ac..178afa3e26 100644 --- a/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx +++ b/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx @@ -1,11 +1,9 @@ --- -title: "@nemo-nb: hide" +title: "Define and Run Custom Python Metrics" description: "" --- - -# Define and Run Custom Python Metrics Custom Python metrics let you score model outputs with deterministic, domain-specific logic that is easier to express in code than in a generic metric or an LLM-as-a-judge prompt. diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index a9ab8ed898..f66164f258 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -1,13 +1,9 @@ --- -title: "@nemo-nb: hide" +title: "Evaluate Response Quality with LLM-as-a-Judge" description: "" --- - - - -# Evaluate Response Quality with LLM-as-a-Judge **LLM-as-a-Judge** is a technique where you use a large language model to evaluate the outputs of another model. Instead of relying solely on automated metrics or manual review, you prompt a capable LLM to score responses based on criteria you define, such as helpfulness, accuracy, or tone. @@ -157,12 +153,12 @@ This tutorial uses `nvidia/nemotron-3-nano-30b-a3b` from NVIDIA Build. ```python from nemo_evaluator_sdk import RunConfig, LLMJudgeMetric from nemo_evaluator_sdk.values import ( - FilesetRef, InferenceParams, JSONScoreParser, Model, RangeScore, ) +from nemo_evaluator.sdk import FilesetRef JUDGE_MODEL_URL = "https://integrate.api.nvidia.com/v1/chat/completions" JUDGE_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b" @@ -368,6 +364,8 @@ The first response is comprehensive and helpful, while the second is unhelpfully Now let's evaluate a larger sample and compare the judge's predictions against human annotations. This tells us how well our judge aligns with human judgment. ```python +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + sample_config = RunConfig( parallelism=1, limit_samples=5, @@ -398,6 +396,7 @@ job_v1 = evaluator.submit( metric=metric_v1_remote, dataset=dataset_ref, config=sample_config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) print(f"Job submitted: {job_v1.name}") ``` @@ -511,6 +510,7 @@ job_v2 = evaluator.submit( metric=metric_v2_remote, dataset=dataset_ref, config=sample_config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) print(f"Job submitted: {job_v2.name}") From 1d156b8de1adf1c0fd9eacadae23c7e4406b19a7 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 23 Jun 2026 16:48:10 -0300 Subject: [PATCH 2/2] docs(evaluator): fix ModelRef submit example config + shorten test name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on #406: - ModelRef submit example now passes config=RunConfigOnlineModel(). A ModelRef target generates outputs online, and _submit_params raises TypeError("ModelRef target requires RunConfigOnlineModel") otherwise — so the example was not runnable. (CodeRabbit flagged this; its suggested fix used params=/the plugin import path — the public submit() keyword is config= and the value type is imported from nemo_evaluator_sdk like the other docs.) - Rename test_submit_exposes_metric_bundle_packager_but_run_does_not to test_packager_param_is_submit_only (reviewer: name too long). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Sandy Chapman --- docs/evaluator/metrics/model-configuration.mdx | 5 +++-- docs/evaluator/test_doc_examples.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index e7f250664a..404378ac3d 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -204,15 +204,16 @@ result = evaluator.run(metric=metric, dataset=dataset, target=model) ### `ModelRef` (supported by `evaluator.submit(...)`) -Durable remote `evaluator.submit(...)` jobs additionally accept a `ModelRef` target. A `ModelRef` names a platform model entity (`workspace/model-name`) and is resolved by the evaluator backend when the job runs, so you do not have to resolve the endpoint yourself. Use this for platform-managed model routing: +Durable remote `evaluator.submit(...)` jobs additionally accept a `ModelRef` target. A `ModelRef` names a platform model entity (`workspace/model-name`) and is resolved by the evaluator backend when the job runs, so you do not have to resolve the endpoint yourself. Use this for platform-managed model routing. A `ModelRef` target generates outputs online, so it requires an online run config (`RunConfigOnlineModel`): ```python -from nemo_evaluator_sdk import ModelRef +from nemo_evaluator_sdk import ModelRef, RunConfigOnlineModel from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager job = evaluator.submit( metric=metric, dataset=dataset, + config=RunConfigOnlineModel(), target=ModelRef(root="default/my-model"), metric_bundle_packager=CloudpickleMetricBundlePackager(), ) diff --git a/docs/evaluator/test_doc_examples.py b/docs/evaluator/test_doc_examples.py index f15222384a..2d9f337a1f 100644 --- a/docs/evaluator/test_doc_examples.py +++ b/docs/evaluator/test_doc_examples.py @@ -74,7 +74,7 @@ def _evaluator() -> Evaluator: return client.evaluator -def test_submit_exposes_metric_bundle_packager_but_run_does_not() -> None: +def test_packager_param_is_submit_only() -> None: """``submit`` takes ``metric_bundle_packager``; ``run`` (local, in-process) does not.""" from nemo_evaluator.sdk import Evaluator