From 50cfeba42db41d03f150791c2fa729b1f4273917 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 26 May 2026 11:10:26 -0300 Subject: [PATCH 1/2] feat(evaluator): bundle metrics for plugin execution Make evaluator plugin job specs bundle-native so backend execution receives MetricBundle payloads, hydrates runtime metrics dynamically, and resolves platform model refs at execution time. Add cloudpickle bundle primitives, plugin compiler/runtime wiring, and local/remote smoke coverage. Signed-off-by: Sandy Chapman --- packages/nemo_platform/pyproject.toml | 1 + plugins/nemo-evaluator/pyproject.toml | 1 + .../src/nemo_evaluator/jobs/compiler.py | 127 +++++ .../src/nemo_evaluator/jobs/evaluate.py | 134 ++--- .../src/nemo_evaluator/jobs/utils.py | 30 +- .../src/nemo_evaluator/sdk/_executor.py | 205 +++++--- .../src/nemo_evaluator/sdk/resources.py | 5 + .../sdk/standalone_sdk/backend.py | 25 +- .../shared/metric_bundles/bundles.py | 242 +++++++++ .../shared/metric_bundles/cloudpickle.py | 86 +++ .../src/nemo_evaluator/tasks/evaluate.py | 19 + .../shared/metric_bundles/test_cloudpickle.py | 252 +++++++++ .../nemo-evaluator/tests/test_evaluate_job.py | 488 +++++++++++------- plugins/nemo-evaluator/tests/test_sdk.py | 233 +++++++-- .../tests/test_sdk_job_resources.py | 14 +- .../tests/test_standalone_sdk_backend.py | 22 +- third_party/licenses.jsonl | 1 + uv.lock | 10 + 18 files changed, 1469 insertions(+), 426 deletions(-) create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py create mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py create mode 100644 plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index 168dfa7ac1..fa59132ddc 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -345,6 +345,7 @@ nemo-data-designer-plugin = [ # Generated from [tool.bundle-package]; do not edit by hand. nemo-evaluator-plugin = [ + "cloudpickle>=3.1.1", "nemo-evaluator-sdk", "nemo-platform-plugin", "nmp-evaluator", diff --git a/plugins/nemo-evaluator/pyproject.toml b/plugins/nemo-evaluator/pyproject.toml index d2ac5303c4..8c5f9fb49d 100644 --- a/plugins/nemo-evaluator/pyproject.toml +++ b/plugins/nemo-evaluator/pyproject.toml @@ -5,6 +5,7 @@ description = "Evaluator plugin scaffold for NeMo Platform." readme = "README.md" requires-python = ">=3.11,<3.14" dependencies = [ + "cloudpickle>=3.1.1", "nemo-evaluator-sdk", "nemo-platform-plugin", "nemo-platform", diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py new file mode 100644 index 0000000000..bd7a712f21 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Plugin-native evaluator job compiler.""" + +from __future__ import annotations + +from nemo_evaluator.jobs.evaluate import EvaluateSpec +from nemo_evaluator_sdk.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel +from nemo_platform_plugin.jobs.api_factory import ( + ContainerSpec, + CPUExecutionProviderSpec, + EnvironmentVariable, + EnvironmentVariableFromSecret, + PlatformJobSpec, + PlatformJobStep, +) +from nemo_platform_plugin.jobs.constants import ( + DEFAULT_JOB_STORAGE_PATH, + EPHEMERAL_TASK_STORAGE_PATH_ENVVAR, + PERSISTENT_JOB_STORAGE_PATH_ENVVAR, +) +from nmp.common.jobs.image import get_qualified_image +from nmp.evaluator.app.values import FilesetRef + +DATASET_DOWNLOAD_STEP_NAME = "dataset-download" +EVALUATE_STEP_NAME = "evaluate" + + +def compile_evaluate_job(spec: EvaluateSpec, *, profile: str | None = None) -> PlatformJobSpec: + """Compile a bundle-native evaluator plugin job.""" + _validate_evaluate_spec(spec) + steps: list[PlatformJobStep] = [] + if isinstance(spec.dataset, FilesetRef): + steps.append(_fileset_download_step(spec.dataset)) + steps.append(_evaluate_step(spec, profile)) + return PlatformJobSpec(steps=steps) + + +def _validate_evaluate_spec(spec: EvaluateSpec) -> None: + if isinstance(spec.target, Model): + if spec.prompt_template is None: + raise ValueError("prompt_template is required when EvaluateSpec.target is a model") + if not isinstance(spec.params, RunConfigOnlineModel): + raise TypeError("model target requires RunConfigOnlineModel") + elif isinstance(spec.target, Agent): + if spec.prompt_template is None: + raise ValueError("prompt_template is required when EvaluateSpec.target is an agent") + if not isinstance(spec.params, RunConfigOnline): + raise TypeError("agent target requires RunConfigOnline") + elif not isinstance(spec.params, RunConfig): + raise TypeError("offline evaluation requires RunConfig") + + +def _fileset_download_step(dataset: FilesetRef) -> PlatformJobStep: + scratch_path = "${" + EPHEMERAL_TASK_STORAGE_PATH_ENVVAR + "}" + target_download_dir = "${" + PERSISTENT_JOB_STORAGE_PATH_ENVVAR + "}/datasets" + return PlatformJobStep( + name=DATASET_DOWNLOAD_STEP_NAME, + executor=CPUExecutionProviderSpec( + provider="cpu", + container=ContainerSpec( + image=get_qualified_image("nmp-cpu-tasks"), + entrypoint=["python", "-m", "nmp.evaluator.tasks.download_fileset"], + command=[ + "--local-dir", + scratch_path, + "--target-dir", + target_download_dir, + "--dataset", + dataset.model_dump_json(), + ], + ), + ), + environment=[ + EnvironmentVariable( + name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR, + value=DEFAULT_JOB_STORAGE_PATH, + ) + ], + ) + + +def _add_secret_ref(secret_refs: dict[str, str], env_name: str, secret_name: str) -> None: + existing = secret_refs.get(env_name) + if existing is not None and existing != secret_name: + raise ValueError(f"conflicting secret references for environment variable {env_name!r}") + secret_refs[env_name] = secret_name + + +def _secret_environment(spec: EvaluateSpec) -> list[EnvironmentVariable]: + environment = [ + EnvironmentVariable( + name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR, + value=DEFAULT_JOB_STORAGE_PATH, + ) + ] + secret_refs: dict[str, str] = {} + for bundle in spec.metrics: + for env_name, secret_ref in bundle.secrets.items(): + _add_secret_ref(secret_refs, env_name, secret_ref.root) + + if isinstance(spec.target, Model | Agent) and spec.target.api_key_secret is not None and spec.target.api_key_env: + _add_secret_ref(secret_refs, spec.target.api_key_env, spec.target.api_key_secret.root) + + environment.extend( + EnvironmentVariable(name=env_name, from_secret=EnvironmentVariableFromSecret(name=secret_name)) + for env_name, secret_name in sorted(secret_refs.items()) + ) + return environment + + +def _evaluate_step(spec: EvaluateSpec, profile: str | None) -> PlatformJobStep: + return PlatformJobStep( + name=EVALUATE_STEP_NAME, + executor=CPUExecutionProviderSpec( + profile=profile or "default", + provider="cpu", + container=ContainerSpec( + image=get_qualified_image("nmp-cpu-tasks"), + entrypoint=["python", "-m"], + command=["nemo_evaluator.tasks.evaluate"], + ), + ), + config=spec.model_dump(mode="json"), + environment=_secret_environment(spec), + ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index 23d92053e3..07fab01dbb 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -6,19 +6,19 @@ from __future__ import annotations import json -from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from typing import Annotated, Any, ClassVar, Self, TypeAlias, cast -from nemo_evaluator.jobs.utils import remote_compile_metric, resolve_run_dataset, resolve_submit_dataset +from nemo_evaluator.jobs.utils import resolve_run_dataset from nemo_evaluator.resolvers import PlatformModelResolver +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, unbundle_metric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayload # noqa: F401 from nemo_evaluator_sdk import Evaluator from nemo_evaluator_sdk.execution._protocols import JobParamsConfigurableMetric from nemo_evaluator_sdk.execution.config import normalize_params from nemo_evaluator_sdk.execution.metric_execution import run_sync -from nemo_evaluator_sdk.metrics.protocol import MetricWithModels -from nemo_evaluator_sdk.metrics.types import MetricsUnion +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithModels from nemo_evaluator_sdk.values import ( Agent, Model, @@ -36,7 +36,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator TargetSpec = Model | Agent -MetricSpec: TypeAlias = MetricsUnion | Annotated[Sequence[MetricsUnion], Field(min_length=1)] +MetricSpec: TypeAlias = Annotated[list[MetricBundle], Field(min_length=1)] EvaluationArtifactResult: TypeAlias = EvaluationResult | BenchmarkEvaluationResult InlineDataset: TypeAlias = Annotated[list[dict[str, object]], Field(min_length=1)] DatasetSpec: TypeAlias = InlineDataset | FilesetRef @@ -66,7 +66,7 @@ class EvaluateSpec(BaseModel): model_config = ConfigDict(extra="forbid") - metric: MetricSpec = Field(description="Inline evaluator SDK metric configuration or benchmark metrics.") + metrics: MetricSpec = Field(description="Bundled metric entities to evaluate.") dataset: DatasetSpec = Field( description="Inline dataset rows or a persisted FilesetRef dataset source to evaluate.", ) @@ -93,37 +93,13 @@ class EvaluateJob(NemoJob): spec_schema: ClassVar[type[BaseModel] | None] = EvaluateSpec job_collection_path: ClassVar[str | None] = "/evaluate/jobs" - @staticmethod - def _write_result_files(result: EvaluationArtifactResult, persistent_dir: Path) -> EvaluationResultFiles: - """Write full, aggregate, and row-level evaluator artifacts.""" - result_payload = result.model_dump(mode="json") - full_result_path = persistent_dir / DEFAULT_FILE_NAME - full_result_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8") - - artifacts_dir = persistent_dir / ARTIFACTS_RESULT_NAME - artifacts_dir.mkdir(parents=True, exist_ok=True) - aggregate_path = artifacts_dir / AGGREGATE_SCORES_FILE_NAME - aggregate_path.write_text(result.aggregate_scores.model_dump_json(indent=2), encoding="utf-8") - row_scores_path = artifacts_dir / ROW_SCORES_FILE_NAME - with row_scores_path.open("w", encoding="utf-8") as f: - for row_score in result.row_scores: - f.write(row_score.model_dump_json() + "\n") - - return EvaluationResultFiles( - full_result=full_result_path, - aggregate_scores=aggregate_path, - row_scores=row_scores_path, - artifacts_dir=artifacts_dir, - ) - @staticmethod async def _resolve_metric_models( - metric: MetricsUnion | Sequence[MetricsUnion], + metrics: list[Metric], resolver: PlatformModelResolver, params: RunConfig | RunConfigOnline | RunConfigOnlineModel, ) -> None: """Resolve ModelRef fields on metric configs before local SDK execution.""" - metrics = metric if isinstance(metric, Sequence) else (metric,) for item in metrics: if isinstance(item, JobParamsConfigurableMetric): item.apply_evaluation_job_params(params) @@ -131,8 +107,7 @@ async def _resolve_metric_models( await item.resolve_models(resolver) @staticmethod - def _unresolved_model_refs(metric: MetricsUnion | Sequence[MetricsUnion]) -> list[str]: - metrics = metric if isinstance(metric, Sequence) else (metric,) + def _unresolved_model_refs(metrics: list[Metric]) -> list[str]: refs = [ model_ref.root for item in metrics @@ -153,57 +128,39 @@ async def compile( profile: str | None = None, options: dict | None = None, ) -> PlatformJobSpec: - """Compile canonical spec using the evaluator service metric job compiler.""" - del workspace, entity_client, job_name, profile, options - canonical_spec = ( - spec.model_copy(deep=True) - if isinstance(spec, EvaluateSpec) - else EvaluateSpec.model_validate(spec.model_dump()) - ) + """Compile canonical spec to a plugin-native evaluator job.""" + del workspace, entity_client, job_name, async_sdk, options + from nemo_evaluator.jobs.compiler import compile_evaluate_job - from nmp.evaluator.app.jobs.metrics import compile_metric_job - from nmp.evaluator.app.values import MetricOfflineJob, MetricOnlineAgentJob, MetricOnlineJob - - dataset, dataset_ref = await resolve_submit_dataset(cast(AsyncNeMoPlatform, async_sdk), canonical_spec.dataset) - params = normalize_params(canonical_spec.params, canonical_spec.target) - await cls._resolve_metric_models(canonical_spec.metric, PlatformModelResolver(async_sdk), params) - metric = remote_compile_metric(canonical_spec.metric) - if isinstance(canonical_spec.target, Model): - if canonical_spec.prompt_template is None: - raise ValueError("prompt_template is required when EvaluateSpec.target is a model") - if not isinstance(params, RunConfigOnlineModel): - raise TypeError("model target requires RunConfigOnlineModel") - metric_job = MetricOnlineJob( - metric=metric, - model=canonical_spec.target, - dataset=dataset, - dataset_ref=dataset_ref, - params=params, - prompt_template=canonical_spec.prompt_template, - ) - elif isinstance(canonical_spec.target, Agent): - if canonical_spec.prompt_template is None: - raise ValueError("prompt_template is required when EvaluateSpec.target is an agent") - if not isinstance(params, RunConfigOnline): - raise TypeError("agent target requires RunConfigOnline") - metric_job = MetricOnlineAgentJob( - metric=metric, - agent=canonical_spec.target, - dataset=dataset, - dataset_ref=dataset_ref, - params=params, - prompt_template=canonical_spec.prompt_template, - ) - else: - if not isinstance(params, RunConfig): - raise TypeError("offline evaluation requires RunConfig") - metric_job = MetricOfflineJob( - metric=metric, - dataset=dataset, - dataset_ref=dataset_ref, - params=params, - ) - return await compile_metric_job(metric_job) + canonical_spec = spec if isinstance(spec, EvaluateSpec) else EvaluateSpec.model_validate(spec.model_dump()) + return compile_evaluate_job(canonical_spec, profile=profile) + + @staticmethod + def _hydrate_metrics(metrics: MetricSpec) -> list[Metric]: + return [unbundle_metric(bundle) for bundle in metrics] + + @staticmethod + def _write_result_files(result: EvaluationArtifactResult, persistent_dir: Path) -> EvaluationResultFiles: + """Write full, aggregate, and row-level evaluator artifacts.""" + result_payload = result.model_dump(mode="json") + full_result_path = persistent_dir / DEFAULT_FILE_NAME + full_result_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8") + + artifacts_dir = persistent_dir / ARTIFACTS_RESULT_NAME + artifacts_dir.mkdir(parents=True, exist_ok=True) + aggregate_path = artifacts_dir / AGGREGATE_SCORES_FILE_NAME + aggregate_path.write_text(result.aggregate_scores.model_dump_json(indent=2), encoding="utf-8") + row_scores_path = artifacts_dir / ROW_SCORES_FILE_NAME + with row_scores_path.open("w", encoding="utf-8") as f: + for row_score in result.row_scores: + f.write(row_score.model_dump_json() + "\n") + + return EvaluationResultFiles( + full_result=full_result_path, + aggregate_scores=aggregate_path, + row_scores=row_scores_path, + artifacts_dir=artifacts_dir, + ) def run(self, config: dict, *, ctx: JobContext, sdk: object | None = None, async_sdk: object | None = None) -> dict: """Run the evaluator job locally and persist its result artifact.""" @@ -211,14 +168,15 @@ def run(self, config: dict, *, ctx: JobContext, sdk: object | None = None, async evaluator = Evaluator() platform_sdk = async_sdk or sdk params = normalize_params(spec.params, spec.target) + metrics = self._hydrate_metrics(spec.metrics) if platform_sdk is None: - unresolved_refs = self._unresolved_model_refs(spec.metric) + unresolved_refs = self._unresolved_model_refs(metrics) if unresolved_refs: raise ValueError( "ModelRef metrics require `sdk` or `async_sdk` for local execution: " + ", ".join(unresolved_refs) ) else: - run_sync(lambda: self._resolve_metric_models(spec.metric, PlatformModelResolver(platform_sdk), params)) + run_sync(lambda: self._resolve_metric_models(metrics, PlatformModelResolver(platform_sdk), params)) dataset = resolve_run_dataset( spec.dataset, ctx=ctx, @@ -231,10 +189,8 @@ def run(self, config: dict, *, ctx: JobContext, sdk: object | None = None, async "target": spec.target, "prompt_template": spec.prompt_template, } - if isinstance(spec.metric, Sequence): - result = evaluator.run_sync(metrics=spec.metric, **common_kwargs) - else: - result = evaluator.run_sync(metrics=cast(MetricsUnion, spec.metric), **common_kwargs) + runtime_metrics = metrics if len(metrics) > 1 else metrics[0] + result = evaluator.run_sync(metrics=runtime_metrics, **common_kwargs) result_files = self._write_result_files(result, ctx.storage.persistent) artifact = ctx.results.save(DEFAULT_RESULT_NAME, result_files.full_result) ctx.results.save(AGGREGATE_SCORES_RESULT_NAME, result_files.aggregate_scores) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py index 13b8ea7f83..c17c7eb0c0 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py @@ -5,41 +5,15 @@ from __future__ import annotations -from collections.abc import Sequence -from typing import Any, cast +from typing import Any from nemo_evaluator_sdk.execution.metric_execution import run_sync -from nemo_evaluator_sdk.metrics.types import MetricsUnion -from nemo_evaluator_sdk.values import DatasetRows from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.job_context import JobContext -from nmp.evaluator.app.datasets.nmp_datasets.fileset import dataset_exists, download_dataset, download_dataset_sync +from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync from nmp.evaluator.app.values import FilesetRef -def remote_compile_metric(metric: MetricsUnion | Sequence[MetricsUnion]) -> MetricsUnion: - """Return the single metric supported by remote service metric-job compilation.""" - if isinstance(metric, Sequence): - raise NotImplementedError("Remote benchmark compilation is not implemented for inline evaluator plugin specs.") - return cast(MetricsUnion, metric) - - -async def resolve_submit_dataset( - async_sdk: AsyncNeMoPlatform, - dataset: list[dict[str, object]] | FilesetRef, -) -> tuple[DatasetRows | FilesetRef, FilesetRef | None]: - """Resolve an evaluator plugin dataset for remote metric-job submission. - - FilesetRef datasets are validated via the async SDK and passed through; - inline rows are wrapped as ``DatasetRows``. - """ - if isinstance(dataset, FilesetRef): - if not await dataset_exists(async_sdk, dataset): - raise ValueError(f"FilesetRef dataset does not exist: {dataset.root}") - return dataset, dataset - return DatasetRows(rows=dataset), None - - def resolve_run_dataset( dataset: list[dict[str, object]] | FilesetRef, *, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index 86bb9f38e1..63855855a2 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -7,12 +7,15 @@ import asyncio from collections.abc import Sequence -from typing import Any, Literal, Protocol, runtime_checkable +from contextlib import asynccontextmanager, contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, AsyncIterator, Iterator, cast import httpx from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec from nemo_evaluator.sdk import http_utils -from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult, local_result_path +from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult from nemo_evaluator.sdk.job_resources import ( AsyncEvaluatorJobResource, EvaluatorJob, @@ -20,11 +23,14 @@ ) from nemo_evaluator.sdk.types import PluginDatasetInput from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, MetricPayloadBundler, bundle_metric +from nemo_evaluator_sdk import Evaluator as SDKEvaluator from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows from nemo_evaluator_sdk.execution.config import EvaluationRequest, normalize_params from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, + DatasetInput, Model, RunConfig, RunConfigOnline, @@ -34,20 +40,27 @@ from nemo_evaluator_sdk.values.results import AggregateFieldName, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.scheduler import NemoJobScheduler +from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync from nmp.evaluator.app.values import FilesetRef _DEFAULT_POLL_INTERVAL_SECONDS = 10.0 _DEFAULT_JOB_TIMEOUT_SECONDS = 3600.0 _DEFAULT_PENDING_TIMEOUT_SECONDS = 600.0 +_ResolvedDataset = DatasetInput | str | Path -@runtime_checkable -class _SerializableMetric(Protocol): - """Metric shape required for evaluator plugin job serialization.""" - def model_dump(self, *, mode: Literal["json"]) -> dict[str, Any]: - """Return the metric as a JSON-serializable payload.""" - ... +class MetricPayloadBundlerPolicyError(RuntimeError): + """Raised when plugin backend metric bundling is not configured.""" + + +def _require_metric_payload_bundler(metric_payload_bundler: MetricPayloadBundler | None) -> MetricPayloadBundler: + if metric_payload_bundler is None: + raise MetricPayloadBundlerPolicyError( + "Bundling runtime metrics for evaluator plugin submission requires an explicit metric_payload_bundler. " + "Pass CloudpickleMetricPayloadBundler() to opt in to cloudpickle metric bundles." + ) + return metric_payload_bundler def _dataset_config(request: EvaluationRequest) -> list[dict[str, Any]] | FilesetRef: @@ -65,10 +78,62 @@ def _dataset_config(request: EvaluationRequest) -> list[dict[str, Any]] | Filese ) -def _build_evaluate_spec(*, metrics: Metric | Sequence[Metric], request: EvaluationRequest) -> EvaluateSpec: +def _fileset_dataset(request: EvaluationRequest) -> FilesetRef: + if not isinstance(request.dataset, FilesetRef): + raise TypeError("request dataset is not a FilesetRef") + if request.dataset_glob_pattern is None: + return request.dataset + if "#" in request.dataset.root: + raise ValueError("dataset_glob_pattern cannot be used when FilesetRef already includes a fragment.") + return request.dataset.with_fragment(request.dataset_glob_pattern) + + +@contextmanager +def _sync_resolved_dataset( + request: EvaluationRequest, platform: NeMoPlatform +) -> Iterator[tuple[_ResolvedDataset, str | None]]: + if not isinstance(request.dataset, FilesetRef): + yield request.dataset, request.dataset_glob_pattern + return + + dataset = _fileset_dataset(request) + with TemporaryDirectory(prefix="nemo-evaluator-fileset-") as temp_dir: + resolved = download_dataset_sync( + sdk=platform, + dataset=dataset, + destination=str(Path(temp_dir) / "dataset"), + ) + yield cast(_ResolvedDataset, resolved), None + + +@asynccontextmanager +async def _async_resolved_dataset( + request: EvaluationRequest, platform: AsyncNeMoPlatform +) -> AsyncIterator[tuple[_ResolvedDataset, str | None]]: + if not isinstance(request.dataset, FilesetRef): + yield request.dataset, request.dataset_glob_pattern + return + + dataset = _fileset_dataset(request) + with TemporaryDirectory(prefix="nemo-evaluator-fileset-") as temp_dir: + resolved = await download_dataset( + sdk=platform, + dataset=dataset, + destination=str(Path(temp_dir) / "dataset"), + ) + yield cast(_ResolvedDataset, resolved), None + + +def _build_evaluate_spec( + *, + metrics: Metric | Sequence[Metric], + request: EvaluationRequest, + metric_payload_bundler: MetricPayloadBundler | None = None, +) -> EvaluateSpec: """Build the evaluator plugin spec shared by local and remote execution.""" + effective_bundler = _require_metric_payload_bundler(metric_payload_bundler) spec = { - "metric": metrics_config(metrics), + "metrics": bundle_metrics_for_spec(metrics, metric_payload_bundler=effective_bundler), "dataset": _dataset_config(request), "params": request.params.model_dump(mode="json") if request.params else None, } @@ -150,9 +215,14 @@ def evaluate_remote( *, metric: Metric, request: EvaluationRequest, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> EvaluationResult: """Submit, poll, and download a remote evaluator plugin metric job.""" - spec = _build_evaluate_spec(metrics=metric, request=request) + spec = _build_evaluate_spec( + metrics=metric, + request=request, + metric_payload_bundler=metric_payload_bundler, + ) job = self.create( spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) @@ -185,15 +255,15 @@ def evaluate( prompt_template=prompt_template, aggregate_fields=aggregate_fields, ) - spec = _build_evaluate_spec(metrics=metric, request=request) - - payload = self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - - result_path = local_result_path(payload) - result = EvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8")) + with _sync_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern): + result = SDKEvaluator().run_sync( + metrics=metric, + dataset=resolved_dataset, + config=request.params, + target=request.target, + dataset_glob_pattern=resolved_pattern, + prompt_template=request.prompt_template, + ) return filter_evaluation_result(result, aggregate_fields) def submit( @@ -205,6 +275,7 @@ def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> EvaluatorJobResource: """Submit a remote evaluator plugin metric job and return the job resource.""" request = EvaluationRequest( @@ -214,7 +285,11 @@ def submit( dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, ) - spec = _build_evaluate_spec(metrics=metric, request=request) + spec = _build_evaluate_spec( + metrics=metric, + request=request, + metric_payload_bundler=metric_payload_bundler, + ) job = self.create( spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) @@ -229,15 +304,15 @@ def evaluate_benchmark( request: EvaluationRequest, ) -> BenchmarkEvaluationResult: """Evaluate multiple metrics through local in-process plugin execution.""" - spec = _build_evaluate_spec(metrics=metrics, request=request) - - payload = self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - - result_path = local_result_path(payload) - result = BenchmarkEvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8")) + with _sync_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern): + result = SDKEvaluator().run_sync( + metrics=metrics, + dataset=resolved_dataset, + config=request.params, + target=request.target, + dataset_glob_pattern=resolved_pattern, + prompt_template=request.prompt_template, + ) return filter_benchmark_result(result, request.aggregate_fields) @@ -319,6 +394,7 @@ async def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> AsyncEvaluatorJobResource: """Submit a remote evaluator plugin metric job and return the job resource.""" request = EvaluationRequest( @@ -328,7 +404,11 @@ async def submit( dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, ) - spec = _build_evaluate_spec(metrics=metric, request=request) + spec = _build_evaluate_spec( + metrics=metric, + request=request, + metric_payload_bundler=metric_payload_bundler, + ) job = await self.create( spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) @@ -341,9 +421,14 @@ async def evaluate_remote( *, metric: Metric, request: EvaluationRequest, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> EvaluationResult: """Submit, poll, and download a remote evaluator plugin metric job.""" - spec = _build_evaluate_spec(metrics=metric, request=request) + spec = _build_evaluate_spec( + metrics=metric, + request=request, + metric_payload_bundler=metric_payload_bundler, + ) job = await self.create( spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) @@ -376,16 +461,15 @@ async def evaluate( prompt_template=prompt_template, aggregate_fields=aggregate_fields, ) - spec = _build_evaluate_spec(metrics=metric, request=request) - - payload = await self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - - result_path = local_result_path(payload) - result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8") - result = EvaluationResult.model_validate_json(result_text) + async with _async_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern): + result = await SDKEvaluator().run( + metrics=metric, + dataset=resolved_dataset, + config=request.params, + target=request.target, + dataset_glob_pattern=resolved_pattern, + prompt_template=request.prompt_template, + ) return filter_evaluation_result(result, aggregate_fields) async def evaluate_benchmark( @@ -395,28 +479,23 @@ async def evaluate_benchmark( request: EvaluationRequest, ) -> BenchmarkEvaluationResult: """Evaluate multiple metrics through local in-process plugin execution.""" - spec = _build_evaluate_spec(metrics=metrics, request=request) - - payload = await self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - - result_path = local_result_path(payload) - result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8") - result = BenchmarkEvaluationResult.model_validate_json(result_text) + async with _async_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern): + result = await SDKEvaluator().run( + metrics=metrics, + dataset=resolved_dataset, + config=request.params, + target=request.target, + dataset_glob_pattern=resolved_pattern, + prompt_template=request.prompt_template, + ) return filter_benchmark_result(result, request.aggregate_fields) -def metric_config(metric: object) -> dict[str, Any]: - """Serialize one metric config for evaluator plugin job submission.""" - if not isinstance(metric, _SerializableMetric): - raise TypeError("metrics must provide model_dump(mode='json') for evaluator plugin execution") - return metric.model_dump(mode="json") - - -def metrics_config(metrics: object | Sequence[object]) -> dict[str, Any] | list[dict[str, Any]]: - """Serialize one metric or a benchmark metric sequence for an evaluator plugin spec.""" +def bundle_metrics_for_spec( + metrics: Metric | Sequence[Metric], *, metric_payload_bundler: MetricPayloadBundler +) -> list[MetricBundle]: + """Bundle one metric or a benchmark metric sequence for an evaluator plugin spec.""" if isinstance(metrics, Sequence) and not isinstance(metrics, (str, bytes)): - return [metric_config(metric) for metric in metrics] - return metric_config(metrics) + metric_sequence = cast(Sequence[Metric], metrics) + return [bundle_metric(metric, metric_payload_bundler) for metric in metric_sequence] + return [bundle_metric(cast(Metric, metrics), metric_payload_bundler)] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index 1339136d81..c5a129eea8 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -24,6 +24,7 @@ RunConfigOnline, RunConfigOnlineModel, ) +from nemo_evaluator.shared.metric_bundles.bundles import MetricPayloadBundler from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, @@ -84,6 +85,7 @@ def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> EvaluatorJobResource: """Submit a metric job through the evaluator plugin executor.""" return self._executor.submit( @@ -93,6 +95,7 @@ def submit( target=target, dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, + metric_payload_bundler=metric_payload_bundler, ) def run( @@ -189,6 +192,7 @@ async def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, + metric_payload_bundler: MetricPayloadBundler | None = None, ) -> AsyncEvaluatorJobResource: """Submit a metric job through the evaluator plugin executor.""" return await self._executor.submit( @@ -198,6 +202,7 @@ async def submit( target=target, dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, + metric_payload_bundler=metric_payload_bundler, ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py index 43af13a789..23e314ac13 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py @@ -10,6 +10,7 @@ from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.sdk.types import ExecutionMode +from nemo_evaluator.shared.metric_bundles.bundles import MetricPayloadBundler from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult @@ -30,6 +31,7 @@ class NMPBackend: resource: Evaluator execution_mode: ExecutionMode = "local" + metric_payload_bundler: MetricPayloadBundler | None = None def evaluate( self, @@ -40,7 +42,11 @@ def evaluate( """Evaluate one metric through local or remote evaluator plugin execution.""" _reject_unsupported_hooks(request) if self.execution_mode == "remote": - return self.resource._executor.evaluate_remote(metric=metric, request=request) + return self.resource._executor.evaluate_remote( + metric=metric, + request=request, + metric_payload_bundler=self.metric_payload_bundler, + ) return self.resource._executor.evaluate( metric=metric, dataset=request.dataset, @@ -61,7 +67,10 @@ def evaluate_benchmark( _reject_unsupported_hooks(request) if self.execution_mode == "remote": raise NotImplementedError("Remote evaluation of benchmarks is not implemented yet.") - return self.resource._executor.evaluate_benchmark(metrics=metrics, request=request) + return self.resource._executor.evaluate_benchmark( + metrics=metrics, + request=request, + ) @dataclass(frozen=True, slots=True) @@ -72,6 +81,7 @@ class AsyncNMPBackend: resource: AsyncEvaluator execution_mode: ExecutionMode = "local" + metric_payload_bundler: MetricPayloadBundler | None = None async def evaluate( self, @@ -82,7 +92,11 @@ async def evaluate( """Evaluate one metric through local or remote evaluator plugin execution.""" _reject_unsupported_hooks(request) if self.execution_mode == "remote": - return await self.resource._executor.evaluate_remote(metric=metric, request=request) + return await self.resource._executor.evaluate_remote( + metric=metric, + request=request, + metric_payload_bundler=self.metric_payload_bundler, + ) return await self.resource._executor.evaluate( metric=metric, dataset=request.dataset, @@ -103,4 +117,7 @@ async def evaluate_benchmark( _reject_unsupported_hooks(request) if self.execution_mode == "remote": raise NotImplementedError("Remote evaluation of benchmarks is not implemented yet.") - return await self.resource._executor.evaluate_benchmark(metrics=metrics, request=request) + return await self.resource._executor.evaluate_benchmark( + metrics=metrics, + request=request, + ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py new file mode 100644 index 0000000000..6ed6c6c0b5 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Backend-neutral metric bundle models and protocols.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Annotated, Any, Literal, Protocol, cast + +from nemo_evaluator_sdk.metrics.protocol import ( + Metric, + MetricOutputSpec, + MetricWithSecrets, +) +from nemo_evaluator_sdk.values.common import SecretRef +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializeAsAny, + StringConstraints, + field_serializer, + field_validator, + model_validator, +) + +BundleMetricTypeName = Annotated[str, StringConstraints(min_length=1)] + + +class MetricBundlingError(ValueError): + """Raised when a metric cannot be bundled or hydrated.""" + + +class MetricMetadata(BaseModel): + """User-facing metadata captured with a bundled metric.""" + + model_config = ConfigDict(extra="allow", revalidate_instances="never") + + description: str | None = None + labels: dict[str, str] = Field(default_factory=dict) + + +class BundledMetricOutputSpec(BaseModel): + """JSON-safe projection of a runtime metric output spec.""" + + model_config = ConfigDict(extra="forbid") + + name: str + description: str | None = None + value_json_schema: dict[str, Any] + + @classmethod + def from_output_spec(cls, output: MetricOutputSpec) -> BundledMetricOutputSpec: + """Capture the serializable contract for one runtime output.""" + return cls( + name=output.name, + description=output.description, + value_json_schema=output.value_json_schema(), + ) + + +class MetricBundlePayload(BaseModel, ABC): + """Base class for concrete Pydantic metric bundle payloads.""" + + @property + @abstractmethod + def kind(self) -> str: + """Payload discriminator used to select the bundler implementation.""" + ... + + @property + @abstractmethod + def digest(self) -> str: + """Format-specific digest for the payload contents.""" + ... + + +class MetricPayloadBundler(Protocol): + """Interface for metric bundle payload implementations.""" + + def bundle(self, metric: Metric) -> MetricBundlePayload: + """Serialize a runtime metric object to a format-specific payload.""" + ... + + def unbundle(self, payload: MetricBundlePayload) -> Metric: + """Hydrate an executable metric from a bundle payload.""" + ... + + +@dataclass(frozen=True) +class _MetricBundleRegistration: + payload_type: type[MetricBundlePayload] + payload_bundler_factory: Callable[[], MetricPayloadBundler] + + +_BUNDLE_REGISTRY: dict[str, _MetricBundleRegistration] = {} + + +def _payload_kind(payload: MetricBundlePayload) -> str: + kind = payload.kind + if not kind: + raise MetricBundlingError("metric bundle payload kind must not be empty") + return kind + + +def register_metric_bundle_kind( + kind: str, + *, + payload_type: type[MetricBundlePayload], + payload_bundler_factory: Callable[[], MetricPayloadBundler], +) -> None: + """Register the payload model and payload bundler factory for a bundle kind.""" + if not kind: + raise ValueError("metric bundle payload kind must not be empty") + _BUNDLE_REGISTRY[kind] = _MetricBundleRegistration( + payload_type=payload_type, + payload_bundler_factory=payload_bundler_factory, + ) + + +class MetricBundle(BaseModel): + """Standalone executable metric bundle entity used by backend execution.""" + + model_config = ConfigDict(extra="forbid") + + bundle_kind: Literal["metric-bundle"] = "metric-bundle" + bundle_format_version: Literal["v1"] = "v1" + metric_type: BundleMetricTypeName + metadata: MetricMetadata = Field(default_factory=MetricMetadata) + outputs: list[BundledMetricOutputSpec] = Field(min_length=1) + secrets: dict[str, SecretRef] = Field(default_factory=dict) + payload: SerializeAsAny[MetricBundlePayload] + + @field_serializer("payload") + def _serialize_payload(self, payload: MetricBundlePayload) -> dict[str, Any]: + value = payload.model_dump(mode="json") + value["kind"] = _payload_kind(payload) + return value + + @field_validator("payload", mode="before") + @classmethod + def _payload_must_have_kind(cls, value: object) -> object: + if isinstance(value, MetricBundlePayload): + return value + if not isinstance(value, Mapping): + raise ValueError("metric bundle payload must be an object") + payload_data = cast(Mapping[str, object], value) + kind = payload_data.get("kind") + if not isinstance(kind, str) or not kind: + raise ValueError("metric bundle payload must include a non-empty kind") + registration = _BUNDLE_REGISTRY.get(kind) + if registration is None: + raise ValueError(f"unsupported metric bundle payload kind: {kind}") + payload_fields = { + field_name: field_value for field_name, field_value in payload_data.items() if field_name != "kind" + } + return registration.payload_type.model_validate(payload_fields) + + @model_validator(mode="after") + def _output_names_must_be_unique(self) -> MetricBundle: + names = [output.name for output in self.outputs] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError(f"duplicate metric output names: {duplicates}") + return self + + +def metric_payload_bundler_for_payload(payload: MetricBundlePayload) -> MetricPayloadBundler: + """Create the payload bundler registered for a metric bundle payload.""" + kind = _payload_kind(payload) + registration = _BUNDLE_REGISTRY.get(kind) + if registration is None: + raise MetricBundlingError(f"unsupported metric bundle payload kind: {kind}") + return registration.payload_bundler_factory() + + +def bundle_metric(metric: Metric, bundler: MetricPayloadBundler) -> MetricBundle: + """Build a standard metric bundle envelope around a format-specific payload.""" + if not isinstance(metric, Metric): + raise MetricBundlingError("object does not satisfy the Metric protocol") + payload = bundler.bundle(metric) + return MetricBundle( + metric_type=validate_metric_type(metric), + metadata=metric_metadata(metric), + outputs=[BundledMetricOutputSpec.from_output_spec(output) for output in metric.output_spec()], + secrets=metric_secrets(metric), + payload=payload, + ) + + +def unbundle_metric(bundle: MetricBundle) -> Metric: + """Hydrate a runtime metric from a standard metric bundle envelope.""" + payload_bundler = metric_payload_bundler_for_payload(bundle.payload) + hydrated_metric = payload_bundler.unbundle(bundle.payload) + _validate_metric_matches_bundle(hydrated_metric, bundle) + return hydrated_metric + + +def _validate_metric_matches_bundle(metric: object, bundle: MetricBundle) -> None: + """Validate that bundle metadata still describes the hydrated metric.""" + if not isinstance(metric, Metric): + raise MetricBundlingError("unbundled object does not satisfy the Metric protocol") + + output_names = [output.name for output in metric.output_spec()] + bundled_output_names = [output.name for output in bundle.outputs] + if output_names != bundled_output_names: + raise MetricBundlingError("unbundled metric output spec does not match bundle metadata") + if validate_metric_type(metric) != bundle.metric_type: + raise MetricBundlingError("unbundled metric type does not match bundle metadata") + + +def validate_metric_type(metric: Metric) -> str: + """Return the runtime metric type after validating the protocol contract.""" + value = metric.type + if not isinstance(value, str): + raise MetricBundlingError("metric type must be a string") + if not value: + raise MetricBundlingError("metric type must not be empty") + return value + + +def metric_metadata(metric: Metric) -> MetricMetadata: + """Capture optional runtime metric metadata.""" + description = getattr(metric, "description", None) + if description is not None and not isinstance(description, str): + raise MetricBundlingError("metric description must be a string when provided") + + raw_labels = getattr(metric, "labels", None) or {} + if not isinstance(raw_labels, Mapping): + raise MetricBundlingError("metric labels must be a mapping when provided") + labels = dict(raw_labels) + return MetricMetadata(description=description, labels=labels) + + +def metric_secrets(metric: Metric) -> dict[str, SecretRef]: + """Capture secret environment mappings needed to execute one metric.""" + if not isinstance(metric, MetricWithSecrets): + return {} + return metric.secrets() diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py new file mode 100644 index 0000000000..af616b64e6 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cloudpickle-backed metric bundle implementation.""" + +from __future__ import annotations + +import hashlib +import pickle +import platform +from typing import Annotated, Literal + +import cloudpickle +from nemo_evaluator.shared.metric_bundles.bundles import ( + MetricBundlePayload, + MetricBundlingError, + MetricPayloadBundler, + register_metric_bundle_kind, +) +from nemo_evaluator_sdk.metrics.protocol import Metric +from pydantic import ConfigDict, Field, computed_field + +NonEmptyBytes = Annotated[bytes, Field(min_length=1)] + + +class CloudpickleMetricPayload(MetricBundlePayload): + """Cloudpickle payload for an executable metric object.""" + + model_config = ConfigDict(extra="ignore", ser_json_bytes="base64", val_json_bytes="base64") + + python_version: str + cloudpickle_version: str + pickle_protocol: int + blob: NonEmptyBytes + + @property + def kind(self) -> Literal["cloudpickle"]: + """Payload discriminator used by the metric bundle registry.""" + return "cloudpickle" + + @computed_field + @property + def digest(self) -> str: + """Digest of the serialized metric payload.""" + return hashlib.sha256(bytes(self.blob)).hexdigest() + + @classmethod + def from_blob(cls, blob: bytes) -> CloudpickleMetricPayload: + """Create a JSON-safe cloudpickle payload from raw bytes.""" + return cls( + python_version=platform.python_version(), + cloudpickle_version=cloudpickle.__version__, + pickle_protocol=pickle.HIGHEST_PROTOCOL, + blob=blob, + ) + + +class CloudpickleMetricPayloadBundler(MetricPayloadBundler): + """Cloudpickle-backed metric payload bundler. + + Cloudpickle bundles execute arbitrary Python code when hydrated. This + implementation is intended for explicit opt-in development/MVP use. + """ + + def bundle(self, metric: Metric) -> MetricBundlePayload: + """Serialize a runtime metric object to a cloudpickle payload.""" + if not isinstance(metric, Metric): + raise MetricBundlingError("object does not satisfy the Metric protocol") + + blob = cloudpickle.dumps(metric, protocol=pickle.HIGHEST_PROTOCOL) + return CloudpickleMetricPayload.from_blob(blob) + + def unbundle(self, payload: MetricBundlePayload) -> Metric: + """Hydrate a metric from a cloudpickle payload.""" + cloudpickle_payload = CloudpickleMetricPayload.model_validate(payload.model_dump(mode="python")) + hydrated_metric = cloudpickle.loads(cloudpickle_payload.blob) + if not isinstance(hydrated_metric, Metric): + raise MetricBundlingError("unbundled object does not satisfy the Metric protocol") + return hydrated_metric + + +register_metric_bundle_kind( + "cloudpickle", + payload_type=CloudpickleMetricPayload, + payload_bundler_factory=CloudpickleMetricPayloadBundler, +) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py new file mode 100644 index 0000000000..8bacacd2a2 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Container entrypoint for evaluator plugin bundle-native jobs.""" + +from __future__ import annotations + +from nemo_evaluator.jobs.evaluate import EvaluateJob +from nemo_platform_plugin.tasks.dispatcher import run_task +from nmp.common.sdk_factory import get_task_sdk + + +def main() -> int: + """Run the evaluator job in a platform-spawned task process.""" + return run_task(EvaluateJob, sdk=get_task_sdk("evaluator")) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py b/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py new file mode 100644 index 0000000000..9449440feb --- /dev/null +++ b/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from typing import cast + +import pytest +from nemo_evaluator.shared.metric_bundles.bundles import ( + MetricBundle, + MetricBundlingError, + bundle_metric, + unbundle_metric, +) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayload, CloudpickleMetricPayloadBundler +from nemo_evaluator_sdk.enums import ModelFormat +from nemo_evaluator_sdk.metrics.bleu import BLEUMetric +from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_evaluator_sdk.metrics.f1 import F1Metric +from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric +from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric +from nemo_evaluator_sdk.metrics.protocol import ( + Metric, + MetricInput, + MetricOutput, + MetricOutputSpec, + MetricResult, +) +from nemo_evaluator_sdk.metrics.ragas import ( + AgentGoalAccuracyMetric, + AnswerAccuracyMetric, + ContextEntityRecallMetric, + ContextPrecisionMetric, + ContextRecallMetric, + ContextRelevanceMetric, + FaithfulnessMetric, + NoiseSensitivityMetric, + ResponseGroundednessMetric, + ResponseRelevancyMetric, + ToolCallAccuracyMetric, + TopicAdherenceMetric, +) +from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric +from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric +from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric +from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric +from nemo_evaluator_sdk.values import Model, SecretRef +from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore, RemoteScore + + +class _CustomMetric: + type = "custom-score" + description = "custom metric" + labels = {"source": "test"} + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("score")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + return MetricResult(outputs=[MetricOutput(name="score", value=1.0)]) + + +class _NotMetric: + pass + + +class _EmptyTypeMetric(_CustomMetric): + type = "" + + +def _judge_model() -> Model: + return Model( + url="https://judge.example.test/v1/chat/completions", + name="judge-model", + format=ModelFormat.OPEN_AI, + ) + + +def _embeddings_model() -> Model: + return Model( + url="https://judge.example.test/v1/embeddings", + name="embedding-model", + format=ModelFormat.OPEN_AI, + ) + + +def _builtin_metric_cases() -> Sequence[tuple[str, Metric]]: + judge_model = _judge_model() + return [ + ("exact_match", ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")), + ("f1", F1Metric(reference="{{item.expected}}", candidate="{{item.output}}")), + ("bleu", BLEUMetric(references=["{{item.expected}}"], candidate="{{item.output}}")), + ("rouge", ROUGEMetric(reference="{{item.expected}}", candidate="{{item.output}}")), + ( + "string_check", + StringCheckMetric( + operation="contains", left_template="{{item.output}}", right_template="{{item.expected}}" + ), + ), + ( + "number_check", + NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}"), + ), + ("tool_calling", ToolCallingMetric(reference="{{item.expected_tool_calls}}")), + ( + "llm_judge", + LLMJudgeMetric( + model=judge_model, + scores=[ + RangeScore( + name="helpfulness", + minimum=1, + maximum=5, + parser=JSONScoreParser(json_path="helpfulness"), + ) + ], + prompt_template="Judge: {{item.expected}} -> {{item.output}}", + ), + ), + ( + "remote", + RemoteMetric( + url="https://remote.example.test", + body={"prompt": "{{item.prompt}}"}, + scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))], + ), + ), + ( + "nemo_agent_toolkit_remote", + NemoAgentToolkitRemoteMetric(url="https://remote.example.test", evaluator_name="nat-quality"), + ), + ("topic_adherence", TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model)), + ("tool_call_accuracy", ToolCallAccuracyMetric()), + ("agent_goal_accuracy", AgentGoalAccuracyMetric(judge_model=judge_model)), + ("answer_accuracy", AnswerAccuracyMetric(judge_model=judge_model)), + ("context_relevance", ContextRelevanceMetric(judge_model=judge_model)), + ("response_groundedness", ResponseGroundednessMetric(judge_model=judge_model)), + ("context_recall", ContextRecallMetric(judge_model=judge_model)), + ("context_precision", ContextPrecisionMetric(judge_model=judge_model)), + ("context_entity_recall", ContextEntityRecallMetric(judge_model=judge_model)), + ( + "response_relevancy", + ResponseRelevancyMetric(judge_model=judge_model, embeddings_model=_embeddings_model()), + ), + ("faithfulness", FaithfulnessMetric(judge_model=judge_model)), + ("noise_sensitivity", NoiseSensitivityMetric(judge_model=judge_model)), + ] + + +def test_cloudpickle_bundler_round_trips_builtin_metric() -> None: + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + bundler = CloudpickleMetricPayloadBundler() + + bundle = bundle_metric(metric, bundler) + hydrated = unbundle_metric(bundle) + + assert bundle.metric_type == "exact-match" + assert bundle.outputs[0].name == "exact-match" + assert isinstance(hydrated, ExactMatchMetric) + + +@pytest.mark.parametrize( + ("case_name", "metric"), _builtin_metric_cases(), ids=[case[0] for case in _builtin_metric_cases()] +) +def test_cloudpickle_bundler_round_trips_every_builtin_metric(case_name: str, metric: Metric) -> None: + bundler = CloudpickleMetricPayloadBundler() + + bundle = bundle_metric(metric, bundler) + restored = MetricBundle.model_validate_json(bundle.model_dump_json()) + hydrated = unbundle_metric(restored) + + assert restored.metric_type + assert restored.outputs + assert [output.name for output in hydrated.output_spec()] == [output.name for output in metric.output_spec()] + assert type(hydrated) is type(metric), case_name + + +def test_cloudpickle_bundler_round_trips_custom_protocol_metric() -> None: + bundler = CloudpickleMetricPayloadBundler() + + bundle = bundle_metric(_CustomMetric(), bundler) + serialized = bundle.model_dump_json() + restored = MetricBundle.model_validate_json(serialized) + hydrated = unbundle_metric(restored) + + assert restored.bundle_kind == "metric-bundle" + assert restored.metric_type == "custom-score" + assert restored.metadata.description == "custom metric" + assert restored.metadata.labels == {"source": "test"} + assert restored.outputs[0].name == "score" + assert isinstance(hydrated, _CustomMetric) + + +def test_cloudpickle_bundler_captures_metric_secrets() -> None: + metric = LLMJudgeMetric( + model=Model( + url="https://judge.example.test/v1/chat/completions", + name="judge-model", + api_key_secret=SecretRef(root="judge-secret"), + format=ModelFormat.OPEN_AI, + ), + scores=[ + RangeScore( + name="helpfulness", + minimum=1, + maximum=5, + parser=JSONScoreParser(json_path="helpfulness"), + ) + ], + ) + + bundle = bundle_metric(metric, CloudpickleMetricPayloadBundler()) + restored = MetricBundle.model_validate_json(bundle.model_dump_json()) + + assert restored.secrets == {"judge_secret": SecretRef(root="judge-secret")} + + +def test_cloudpickle_bundler_captures_digest_and_payload_metadata() -> None: + bundle = bundle_metric(_CustomMetric(), CloudpickleMetricPayloadBundler()) + payload = CloudpickleMetricPayload.model_validate(bundle.payload) + serialized_payload = cast(dict[str, object], bundle.model_dump(mode="json")["payload"]) + + assert payload.digest == hashlib.sha256(bytes(payload.blob)).hexdigest() + assert serialized_payload["digest"] == payload.digest + assert payload.kind == "cloudpickle" + assert serialized_payload["kind"] == "cloudpickle" + assert payload.python_version + assert payload.cloudpickle_version + assert payload.pickle_protocol > 0 + assert bundle.outputs[0].value_json_schema["title"] == "ContinuousScore" + + +def test_cloudpickle_bundler_rejects_non_metric_object() -> None: + with pytest.raises(MetricBundlingError, match="Metric protocol"): + bundle_metric(cast(Metric, _NotMetric()), CloudpickleMetricPayloadBundler()) + + +def test_cloudpickle_bundler_rejects_empty_metric_type() -> None: + with pytest.raises(MetricBundlingError, match="metric type must not be empty"): + bundle_metric(_EmptyTypeMetric(), CloudpickleMetricPayloadBundler()) + + +def test_cloudpickle_bundler_hydrates_from_payload_without_bundle_envelope() -> None: + bundler = CloudpickleMetricPayloadBundler() + bundle = bundle_metric(_CustomMetric(), bundler) + + hydrated = bundler.unbundle(bundle.payload) + + assert isinstance(hydrated, _CustomMetric) diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index b535fcb0b8..3cf54c5152 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -8,7 +8,7 @@ import json from pathlib import Path from types import SimpleNamespace -from typing import Any, cast +from typing import Any, Literal, cast import pytest from nemo_evaluator.cli import EvaluatorPluginCLI @@ -22,10 +22,31 @@ EvaluateSpec, ) from nemo_evaluator.resolvers import PlatformModelResolver, _parse_required_workspace_name +from nemo_evaluator.shared.metric_bundles.bundles import ( + MetricBundle, + MetricBundlePayload, + MetricPayloadBundler, + bundle_metric, + register_metric_bundle_kind, + unbundle_metric, +) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler +from nemo_evaluator.tasks.evaluate import main as evaluate_task_main from nemo_evaluator_sdk.enums import AgentFormat +from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +from nemo_evaluator_sdk.metrics.f1 import F1Metric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric -from nemo_evaluator_sdk.metrics.protocol import MetricOutput, MetricResult -from nemo_evaluator_sdk.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult +from nemo_evaluator_sdk.values import ( + Agent, + AggregatedMetricResult, + EvaluationResult, + Model, + RunConfig, + RunConfigOnline, + RunConfigOnlineModel, + SecretRef, +) from nemo_evaluator_sdk.values.models import ModelRef from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore from nemo_platform.types.jobs.platform_job_spec import PlatformJobSpec @@ -34,18 +55,16 @@ from nemo_platform_plugin.job_results import LocalJobResults from nemo_platform_plugin.scheduler import NemoJobScheduler from nmp.evaluator.app.values import FilesetRef -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pytest_mock import MockerFixture from typer.testing import CliRunner def _exact_match_spec() -> dict: return { - "metric": { - "type": "exact-match", - "reference": "{{item.expected}}", - "candidate": "{{item.model_output}}", - }, + "metrics": [ + _bundle_payload(ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}")) + ], "dataset": [ {"expected": "blue", "model_output": "Blue"}, {"expected": "Jupiter", "model_output": "Saturn"}, @@ -54,16 +73,15 @@ def _exact_match_spec() -> dict: } +def _bundle_payload(metric) -> dict[str, Any]: + return bundle_metric(metric, CloudpickleMetricPayloadBundler()).model_dump(mode="json") + + def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None: step = job_spec.steps[0] container = cast(Any, step.executor).container - assert container.entrypoint == ["python", "-m", "nmp.evaluator.tasks.evaluate_metric"] - command = container.command - assert command is not None - assert command == [ - "--progress-tracking-url", - "${NMP_JOBS_URL}/apis/jobs/v2/workspaces/${NEMO_JOB_WORKSPACE}/jobs/${NEMO_JOB_ID}/status-details", - ] + assert container.entrypoint == ["python", "-m"] + assert container.command == ["nemo_evaluator.tasks.evaluate"] def _load_cli_run_payload(output: str) -> dict[str, Any]: @@ -83,6 +101,11 @@ def _make_job_context(tmp_path: Path) -> JobContext: ) +def _empty_evaluation_result() -> EvaluationResult: + """Return an SDK result object suitable for runner delegation tests.""" + return EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) + + def _assert_saved_result_artifact( run_result: dict[str, Any], ctx: JobContext, result_payload: dict[str, object] ) -> None: @@ -95,24 +118,9 @@ def _assert_saved_result_artifact( assert json.loads(result_path.read_text(encoding="utf-8")) == result_payload artifact_path = Path(run_result["artifact"]["artifact_url"].removeprefix("file://")) assert json.loads(artifact_path.read_text(encoding="utf-8")) == result_payload - results_dir = ctx.storage.persistent / "results" - assert ( - json.loads((results_dir / AGGREGATE_SCORES_RESULT_NAME).read_text(encoding="utf-8")) - == result_payload["aggregate_scores"] - ) - assert (results_dir / ROW_SCORES_RESULT_NAME).read_text(encoding="utf-8") == "" - assert (results_dir / ARTIFACTS_RESULT_NAME).is_dir() - - -def _mock_evaluation_result(mocker: MockerFixture, result_payload: dict[str, object]) -> Any: - """Return an evaluator result mock with platform result artifact surfaces.""" - result = mocker.Mock() - result.model_dump.return_value = result_payload - aggregate_scores = mocker.Mock() - aggregate_scores.model_dump_json.return_value = json.dumps(result_payload["aggregate_scores"], indent=2) - result.aggregate_scores = aggregate_scores - result.row_scores = [] - return result + assert (ctx.storage.persistent / "results" / AGGREGATE_SCORES_RESULT_NAME).exists() + assert (ctx.storage.persistent / "results" / ROW_SCORES_RESULT_NAME).exists() + assert (ctx.storage.persistent / "results" / ARTIFACTS_RESULT_NAME).is_dir() def _load_artifact_payload(run_result: dict[str, Any]) -> dict[str, Any]: @@ -121,6 +129,66 @@ def _load_artifact_payload(run_result: dict[str, Any]) -> dict[str, Any]: return cast(dict[str, Any], json.loads(artifact_path.read_text(encoding="utf-8"))) +class _StaticMetric: + def __init__(self, metric_type: str) -> None: + self._metric_type = metric_type + + @property + def type(self) -> str: + return self._metric_type + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("score")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + del input + return MetricResult(outputs=[MetricOutput(name="score", value=1.0)]) + + +class _StaticMetricPayload(MetricBundlePayload): + @property + def kind(self) -> Literal["test-static"]: + return "test-static" + + @property + def digest(self) -> str: + return "test-static-digest" + + +class _StrictMetricPayload(MetricBundlePayload): + model_config = ConfigDict(extra="forbid") + + @property + def kind(self) -> Literal["test-strict"]: + return "test-strict" + + @property + def digest(self) -> str: + return "test-strict-digest" + + +class _StaticMetricPayloadBundler(MetricPayloadBundler): + def bundle(self, metric: Metric) -> MetricBundlePayload: + del metric + return _StaticMetricPayload() + + def unbundle(self, payload: MetricBundlePayload) -> Metric: + del payload + return _StaticMetric("test-static") + + +register_metric_bundle_kind( + "test-static", + payload_type=_StaticMetricPayload, + payload_bundler_factory=_StaticMetricPayloadBundler, +) +register_metric_bundle_kind( + "test-strict", + payload_type=_StrictMetricPayload, + payload_bundler_factory=_StaticMetricPayloadBundler, +) + + class _FakeModels: def __init__(self) -> None: self.retrieved: list[tuple[str, str]] = [] @@ -223,6 +291,32 @@ def test_parse_required_workspace_name_rejects_extra_separator() -> None: _parse_required_workspace_name("default/judge/extra", label="ModelRef", expected_format="workspace/model_name") +def test_evaluate_job_hydrates_mixed_bundle_kinds_by_payload_kind() -> None: + """Execution-side hydration dispatches per bundle instead of assuming one bundler.""" + cloudpickle_bundle = bundle_metric( + ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + CloudpickleMetricPayloadBundler(), + ) + static_bundle = bundle_metric(_StaticMetric("test-static"), _StaticMetricPayloadBundler()) + + metrics = EvaluateJob._hydrate_metrics([cloudpickle_bundle, static_bundle]) + + assert [metric.type for metric in metrics] == ["exact-match", "test-static"] + + +def test_metric_bundle_validation_strips_payload_kind_before_payload_validation() -> None: + """Payload kind is the registry discriminator, not a concrete payload model field.""" + bundle = MetricBundle.model_validate( + { + "metric_type": "test-strict", + "outputs": [{"name": "score", "value_json_schema": {"type": "number"}}], + "payload": {"kind": "test-strict"}, + } + ) + + assert isinstance(bundle.payload, _StrictMetricPayload) + + def test_evaluate_job_resolves_metric_model_refs_before_sdk_run( tmp_path: Path, mocker: MockerFixture, @@ -240,7 +334,7 @@ async def compute_scores(metric: LLMJudgeMetric, input) -> MetricResult: ctx = _make_job_context(tmp_path) run_result = EvaluateJob().run( { - "metric": _llm_judge_ref_metric().model_dump(mode="json"), + "metrics": [_bundle_payload(_llm_judge_ref_metric())], "dataset": [{"output_text": "hello"}], }, ctx=ctx, @@ -252,19 +346,6 @@ async def compute_scores(metric: LLMJudgeMetric, input) -> MetricResult: assert payload["aggregate_scores"]["scores"][0]["mean"] == 1.0 -def test_evaluate_job_rejects_model_refs_without_platform_sdk(tmp_path: Path) -> None: - ctx = _make_job_context(tmp_path) - - with pytest.raises(ValueError, match="ModelRef metrics require `sdk` or `async_sdk`"): - EvaluateJob().run( - { - "metric": _llm_judge_ref_metric().model_dump(mode="json"), - "dataset": [{"output_text": "hello"}], - }, - ctx=ctx, - ) - - async def test_evaluate_job_compile_produces_cpu_task_step() -> None: spec = EvaluateSpec.model_validate(_exact_match_spec()) compiled = await EvaluateJob.compile( @@ -277,20 +358,21 @@ async def test_evaluate_job_compile_produces_cpu_task_step() -> None: job_spec = PlatformJobSpec.model_validate(compiled) assert len(job_spec.steps) == 1 step = job_spec.steps[0] - assert step.name == "evaluation" + assert step.name == "evaluate" _assert_metric_step_entrypoint(job_spec) assert step.config is not None config = cast(dict[str, Any], step.config) - assert config["metric"]["type"] == "exact-match" - assert config["dataset"]["rows"] == _exact_match_spec()["dataset"] + assert config["metrics"][0]["bundle_kind"] == "metric-bundle" + assert config["metrics"][0]["metric_type"] == "exact-match" + assert config["dataset"] == _exact_match_spec()["dataset"] -async def test_evaluate_job_compile_resolves_metric_model_refs_before_remote_job() -> None: +async def test_evaluate_job_compile_preserves_bundled_metric_model_refs_for_runtime_resolution() -> None: compiled = await EvaluateJob.compile( workspace="default", spec=EvaluateSpec.model_validate( { - "metric": _llm_judge_ref_metric().model_dump(mode="json"), + "metrics": [_bundle_payload(_llm_judge_ref_metric())], "dataset": [{"output_text": "hello"}], } ), @@ -301,9 +383,11 @@ async def test_evaluate_job_compile_resolves_metric_model_refs_before_remote_job job_spec = PlatformJobSpec.model_validate(compiled) config = cast(dict[str, Any], job_spec.steps[0].config) - assert config["metric"]["model"]["name"] == "judge" - assert config["metric"]["model"]["url"] == "https://igw.example.test/v1/chat/completions" - assert config["metric"]["model"]["host_url"] == "http://nim.example.test:8000" + metric_bundle = MetricBundle.model_validate(config["metrics"][0]) + metric = unbundle_metric(metric_bundle) + assert isinstance(metric, LLMJudgeMetric) + assert isinstance(metric.model, ModelRef) + assert metric.model.root == "default/judge" async def test_evaluate_job_compile_produces_online_model_job() -> None: @@ -328,7 +412,7 @@ async def test_evaluate_job_compile_produces_online_model_job() -> None: step = job_spec.steps[0] config = cast(dict[str, Any], step.config) _assert_metric_step_entrypoint(job_spec) - assert config["model"]["name"] == "test-model" + assert config["target"]["name"] == "test-model" assert config["prompt_template"] == "Question: {{item.question}}" assert config["params"]["parallelism"] == 3 @@ -360,10 +444,57 @@ async def test_evaluate_job_compile_produces_online_agent_job() -> None: step = job_spec.steps[0] config = cast(dict[str, Any], step.config) _assert_metric_step_entrypoint(job_spec) - assert config["agent"]["name"] == "test-agent" + assert config["target"]["name"] == "test-agent" assert config["prompt_template"] == {"question": "{{item.question}}"} +async def test_evaluate_job_compile_injects_metric_and_target_secrets() -> None: + secret_ref = SecretRef(root="NVIDIA_BUILD_API_KEY") + spec = EvaluateSpec.model_validate( + { + **_exact_match_spec(), + "metrics": [ + _bundle_payload( + LLMJudgeMetric( + model=Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/nemotron-3-super-120b-a12b", + api_key_secret=secret_ref, + ), + scores=[ + RangeScore( + name="quality", + minimum=1, + maximum=5, + parser=JSONScoreParser(json_path="quality"), + ), + ], + ) + ) + ], + "target": Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/nemotron-3-super-120b-a12b", + api_key_secret=secret_ref, + ), + "params": RunConfigOnlineModel(parallelism=3), + "prompt_template": "Question: {{item.question}}", + } + ) + + compiled = await EvaluateJob.compile( + workspace="default", + spec=spec, + entity_client=object(), + job_name=None, + async_sdk=object(), + ) + + step = PlatformJobSpec.model_validate(compiled).steps[0] + secrets = {env.name: env.from_secret.name for env in step.environment or [] if env.from_secret} + assert secrets == {"NVIDIA_BUILD_API_KEY": "NVIDIA_BUILD_API_KEY"} + + class TestEvaluateSpec: """Validation coverage for evaluator job specs.""" @@ -376,7 +507,7 @@ def test_rejects_empty_dataset(self) -> None: } ) - def test_rejects_legacy_metrics_field(self) -> None: + def test_rejects_legacy_metric_config(self) -> None: with pytest.raises(ValueError, match="metrics|Extra inputs are not permitted"): EvaluateSpec.model_validate( { @@ -389,44 +520,50 @@ def test_rejects_legacy_metrics_field(self) -> None: } ) + def test_rejects_singular_metric_field(self) -> None: + with pytest.raises(ValueError, match="metrics|Extra inputs are not permitted"): + EvaluateSpec.model_validate( + { + "metric": _exact_match_spec()["metrics"][0], + "dataset": _exact_match_spec()["dataset"], + } + ) + def test_accepts_metrics_sequence(self) -> None: spec = EvaluateSpec.model_validate( { **_exact_match_spec(), - "metric": [ - _exact_match_spec()["metric"], - { - "type": "f1", - "reference": "{{item.expected}}", - "candidate": "{{item.model_output}}", - }, + "metrics": [ + _exact_match_spec()["metrics"][0], + _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")), ], } ) - assert isinstance(spec.metric, list) - assert [metric.type.value for metric in spec.metric] == ["exact-match", "f1"] + assert [metric.metric_type for metric in spec.metrics] == ["exact-match", "f1"] def test_accepts_uppercase_api_key_secret_refs_for_llm_judge_and_target(self) -> None: spec = EvaluateSpec.model_validate( { - "metric": { - "type": "llm-judge", - "model": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_BUILD_API_KEY", - "format": "nim", - }, - "scores": [ - { - "name": "quality", - "minimum": 1, - "maximum": 5, - "parser": {"type": "json", "json_path": "quality"}, - }, - ], - }, + "metrics": [ + _bundle_payload( + LLMJudgeMetric( + model=Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/nemotron-3-super-120b-a12b", + api_key_secret=SecretRef(root="NVIDIA_BUILD_API_KEY"), + ), + scores=[ + RangeScore( + name="quality", + minimum=1, + maximum=5, + parser=JSONScoreParser(json_path="quality"), + ), + ], + ) + ) + ], "dataset": [{"prompt": "Hello", "model_output": "Hi"}], "target": { "url": "https://integrate.api.nvidia.com/v1/chat/completions", @@ -437,12 +574,9 @@ def test_accepts_uppercase_api_key_secret_refs_for_llm_judge_and_target(self) -> } ) - assert isinstance(spec.metric, LLMJudgeMetric) - assert isinstance(spec.metric.model, Model) assert isinstance(spec.target, Model) - assert spec.metric.model.api_key_secret is not None assert spec.target.api_key_secret is not None - assert spec.metric.model.api_key_secret.root == "NVIDIA_BUILD_API_KEY" + assert spec.metrics[0].metric_type == "llm-judge" assert spec.target.api_key_secret.root == "NVIDIA_BUILD_API_KEY" def test_rejects_extra_fields(self) -> None: @@ -481,7 +615,7 @@ async def test_accepts_equivalent_base_model_spec(self) -> None: class EquivalentSpec(BaseModel): """Spec shape used to verify compile canonicalizes BaseModel inputs.""" - metric: dict[str, object] + metrics: list[dict[str, object]] dataset: list[dict[str, object]] params: dict[str, object] | None = None @@ -496,33 +630,32 @@ class EquivalentSpec(BaseModel): job_spec = PlatformJobSpec.model_validate(compiled) step = job_spec.steps[0] config = cast(dict[str, Any], step.config) - assert config["metric"]["type"] == "exact-match" - assert config["dataset"]["rows"] == _exact_match_spec()["dataset"] + assert config["metrics"][0]["bundle_kind"] == "metric-bundle" + assert config["metrics"][0]["metric_type"] == "exact-match" + assert config["dataset"] == _exact_match_spec()["dataset"] assert config["params"]["parallelism"] == 2 - async def test_rejects_remote_compile_for_metrics_sequence(self) -> None: + async def test_accepts_metrics_sequence(self) -> None: spec = EvaluateSpec.model_validate( { **_exact_match_spec(), - "metric": [ - _exact_match_spec()["metric"], - { - "type": "f1", - "reference": "{{item.expected}}", - "candidate": "{{item.model_output}}", - }, + "metrics": [ + _exact_match_spec()["metrics"][0], + _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")), ], } ) - with pytest.raises(NotImplementedError, match="Remote benchmark.*not implemented"): - await EvaluateJob.compile( - workspace="default", - spec=spec, - entity_client=object(), - job_name=None, - async_sdk=object(), - ) + compiled = await EvaluateJob.compile( + workspace="default", + spec=spec, + entity_client=object(), + job_name=None, + async_sdk=object(), + ) + + config = cast(dict[str, Any], PlatformJobSpec.model_validate(compiled).steps[0].config) + assert [metric["metric_type"] for metric in config["metrics"]] == ["exact-match", "f1"] @pytest.mark.parametrize( ("target", "expected_message"), @@ -611,49 +744,25 @@ async def test_rejects_wrong_offline_param_type(self, mocker: MockerFixture) -> async_sdk=object(), ) - async def test_fileset_ref_dataset_validates_and_compiles_with_download_step(self, mocker: MockerFixture) -> None: + async def test_fileset_ref_dataset_compiles_into_bundle_native_step(self) -> None: dataset = FilesetRef(root="default/helpsteer2#validation/*.jsonl") - dataset_exists = mocker.patch( - "nemo_evaluator.jobs.utils.dataset_exists", - new=mocker.AsyncMock(return_value=True), - create=True, - ) - async_sdk = object() compiled = await EvaluateJob.compile( workspace="default", spec=EvaluateSpec.model_validate({**_exact_match_spec(), "dataset": dataset}), entity_client=object(), job_name=None, - async_sdk=async_sdk, + async_sdk=object(), ) job_spec = PlatformJobSpec.model_validate(compiled) - assert [step.name for step in job_spec.steps] == ["dataset-download", "evaluation"] + assert [step.name for step in job_spec.steps] == ["dataset-download", "evaluate"] + download_step = job_spec.steps[0] + download_container = cast(Any, download_step.executor).container + assert download_container.entrypoint == ["python", "-m", "nmp.evaluator.tasks.download_fileset"] + assert download_container.command[-2:] == ["--dataset", dataset.model_dump_json()] config = cast(dict[str, Any], job_spec.steps[1].config) assert config["dataset"] == dataset.root - assert config["dataset_ref"] == dataset.root - dataset_exists.assert_awaited_once_with(async_sdk, dataset) - - async def test_fileset_ref_dataset_compile_raises_when_dataset_does_not_exist(self, mocker: MockerFixture) -> None: - dataset = FilesetRef(root="default/missing") - dataset_exists = mocker.patch( - "nemo_evaluator.jobs.utils.dataset_exists", - new=mocker.AsyncMock(return_value=False), - create=True, - ) - async_sdk = object() - - with pytest.raises(ValueError, match="FilesetRef dataset does not exist: default/missing"): - await EvaluateJob.compile( - workspace="default", - spec=EvaluateSpec.model_validate({**_exact_match_spec(), "dataset": dataset}), - entity_client=object(), - job_name=None, - async_sdk=async_sdk, - ) - - dataset_exists.assert_awaited_once_with(async_sdk, dataset) class TestEvaluateJobRun: @@ -692,8 +801,8 @@ def test_delegates_to_sdk_evaluator( tmp_path: Path, mocker: MockerFixture, ) -> None: - result_payload = {"aggregate_scores": {"scores": []}} - result = _mock_evaluation_result(mocker, result_payload) + result = _empty_evaluation_result() + result_payload = result.model_dump(mode="json") evaluator = mocker.Mock() evaluator.run_sync.return_value = result evaluator_cls = mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator) @@ -715,34 +824,24 @@ def test_delegates_to_sdk_evaluator( assert "result" not in run_result _assert_saved_result_artifact(run_result, ctx, result_payload) evaluator_cls.assert_called_once_with() - evaluator.run_sync.assert_called_once_with( - metrics=expected_spec.metric, - dataset=expected_spec.dataset, - config=expected_config, - target=expected_spec.target, - prompt_template=expected_spec.prompt_template, - ) - result.model_dump.assert_called_once_with(mode="json") + call_kwargs = evaluator.run_sync.call_args.kwargs + assert isinstance(call_kwargs["metrics"], ExactMatchMetric) + assert call_kwargs["dataset"] == expected_spec.dataset + assert call_kwargs["config"] == expected_config + assert call_kwargs["target"] == expected_spec.target + assert call_kwargs["prompt_template"] == expected_spec.prompt_template def test_delegates_metrics_sequence_to_sdk_evaluator(self, tmp_path: Path, mocker: MockerFixture) -> None: - result_payload = { - "row_scores": [], - "aggregate_scores": {"scores": []}, - "per_metric": {}, - } - result = _mock_evaluation_result(mocker, result_payload) + result = _empty_evaluation_result() + result_payload = result.model_dump(mode="json") evaluator = mocker.Mock() evaluator.run_sync.return_value = result evaluator_cls = mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator) config = { **_exact_match_spec(), - "metric": [ - _exact_match_spec()["metric"], - { - "type": "f1", - "reference": "{{item.expected}}", - "candidate": "{{item.model_output}}", - }, + "metrics": [ + _exact_match_spec()["metrics"][0], + _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")), ], } expected_spec = EvaluateSpec.model_validate(config) @@ -757,20 +856,18 @@ def test_delegates_metrics_sequence_to_sdk_evaluator(self, tmp_path: Path, mocke assert "result" not in run_result _assert_saved_result_artifact(run_result, ctx, result_payload) evaluator_cls.assert_called_once_with() - evaluator.run_sync.assert_called_once_with( - metrics=expected_spec.metric, - dataset=expected_spec.dataset, - config=expected_spec.params, - target=expected_spec.target, - prompt_template=expected_spec.prompt_template, - ) - result.model_dump.assert_called_once_with(mode="json") + call_kwargs = evaluator.run_sync.call_args.kwargs + assert [metric.type.value for metric in call_kwargs["metrics"]] == ["exact-match", "f1"] + assert call_kwargs["dataset"] == expected_spec.dataset + assert call_kwargs["config"] == expected_spec.params + assert call_kwargs["target"] == expected_spec.target + assert call_kwargs["prompt_template"] == expected_spec.prompt_template def test_downloads_fileset_ref_dataset_and_passes_path_to_sdk_evaluator( self, tmp_path: Path, mocker: MockerFixture ) -> None: - result_payload = {"aggregate_scores": {"scores": []}} - result = _mock_evaluation_result(mocker, result_payload) + result = _empty_evaluation_result() + result_payload = result.model_dump(mode="json") evaluator = mocker.Mock() evaluator.run_sync.return_value = result mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator) @@ -795,20 +892,18 @@ def test_downloads_fileset_ref_dataset_and_passes_path_to_sdk_evaluator( destination=str(ctx.storage.persistent / "dataset"), ) download_dataset_sync.assert_not_called() - evaluator.run_sync.assert_called_once_with( - metrics=EvaluateSpec.model_validate(config).metric, - dataset=downloaded_path, - config=EvaluateSpec.model_validate(config).params, - target=None, - prompt_template=None, - ) - result.model_dump.assert_called_once_with(mode="json") + call_kwargs = evaluator.run_sync.call_args.kwargs + assert isinstance(call_kwargs["metrics"], ExactMatchMetric) + assert call_kwargs["dataset"] == downloaded_path + assert call_kwargs["config"] == EvaluateSpec.model_validate(config).params + assert call_kwargs["target"] is None + assert call_kwargs["prompt_template"] is None def test_downloads_fileset_ref_dataset_with_sync_sdk_and_passes_path_to_sdk_evaluator( self, tmp_path: Path, mocker: MockerFixture ) -> None: - result_payload = {"aggregate_scores": {"scores": []}} - result = _mock_evaluation_result(mocker, result_payload) + result = _empty_evaluation_result() + result_payload = result.model_dump(mode="json") evaluator = mocker.Mock() evaluator.run_sync.return_value = result mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator) @@ -833,11 +928,24 @@ def test_downloads_fileset_ref_dataset_with_sync_sdk_and_passes_path_to_sdk_eval dataset=dataset, destination=str(ctx.storage.persistent / "dataset"), ) - evaluator.run_sync.assert_called_once_with( - metrics=EvaluateSpec.model_validate(config).metric, - dataset=downloaded_path, - config=EvaluateSpec.model_validate(config).params, - target=None, - prompt_template=None, - ) - result.model_dump.assert_called_once_with(mode="json") + call_kwargs = evaluator.run_sync.call_args.kwargs + assert isinstance(call_kwargs["metrics"], ExactMatchMetric) + assert call_kwargs["dataset"] == downloaded_path + assert call_kwargs["config"] == EvaluateSpec.model_validate(config).params + assert call_kwargs["target"] is None + assert call_kwargs["prompt_template"] is None + + +class TestEvaluateTask: + """Coverage for the compiled container task entrypoint.""" + + def test_main_dispatches_evaluate_job_with_task_sdk(self, mocker: MockerFixture) -> None: + sdk = object() + get_task_sdk = mocker.patch("nemo_evaluator.tasks.evaluate.get_task_sdk", return_value=sdk) + run_task = mocker.patch("nemo_evaluator.tasks.evaluate.run_task", return_value=0) + + exit_code = evaluate_task_main() + + assert exit_code == 0 + get_task_sdk.assert_called_once_with("evaluator") + run_task.assert_called_once_with(EvaluateJob, sdk=sdk) diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py index 401bac080c..aef0eb2dfe 100644 --- a/plugins/nemo-evaluator/tests/test_sdk.py +++ b/plugins/nemo-evaluator/tests/test_sdk.py @@ -5,7 +5,7 @@ from __future__ import annotations -from collections.abc import Sequence +from pathlib import Path from typing import Any, cast from unittest.mock import AsyncMock, MagicMock @@ -14,18 +14,26 @@ from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec from nemo_evaluator.sdk import http_utils from nemo_evaluator.sdk._executor import ( + MetricPayloadBundlerPolicyError, _AsyncEvaluatorPluginExecutor, _build_evaluate_spec, _SyncEvaluatorPluginExecutor, - metric_config, + bundle_metrics_for_spec, ) from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator -from nemo_evaluator_sdk.enums import MetricType +from nemo_evaluator.shared.metric_bundles.bundles import ( + MetricBundle, + MetricBundlePayload, + MetricBundlingError, + MetricPayloadBundler, + bundle_metric, +) +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric -from nemo_evaluator_sdk.metrics.types import MetricsUnion +from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import Model, RunConfig, RunConfigOnlineModel from nemo_evaluator_sdk.values.results import AggregatedMetricResult, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform @@ -34,8 +42,18 @@ from pydantic import ValidationError from pytest_mock import MockerFixture +_EXACT_MATCH_METRIC = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") _EXACT_MATCH_SPEC = { - "metric": { + "metrics": [ + bundle_metric( + _EXACT_MATCH_METRIC, + CloudpickleMetricPayloadBundler(), + ).model_dump(mode="json") + ], + "dataset": [{"expected": "a", "output": "a"}], +} +_LEGACY_EXACT_MATCH_SPEC = { + "metrics": { "type": "exact-match", "reference": "{{item.expected}}", "candidate": "{{item.output}}", @@ -46,11 +64,27 @@ _EXACT_MATCH_EVALUATE_SPEC_JSON = _EXACT_MATCH_EVALUATE_SPEC.model_dump(mode="json") -def _single_metric(spec: EvaluateSpec) -> MetricsUnion: +def _single_metric(spec: EvaluateSpec) -> MetricBundle: """Return the single metric from an evaluator job spec.""" - if isinstance(spec.metric, Sequence): + if len(spec.metrics) != 1: raise AssertionError("Expected a single metric spec.") - return spec.metric + return spec.metrics[0] + + +class _RecordingMetricPayloadBundler(MetricPayloadBundler): + """Test bundler that records all runtime metrics selected for bundling.""" + + def __init__(self) -> None: + self.metrics: list[Metric] = [] + self._delegate = CloudpickleMetricPayloadBundler() + + def bundle(self, metric: Metric) -> MetricBundlePayload: + self.metrics.append(metric) + return self._delegate.bundle(metric) + + def unbundle(self, payload: MetricBundlePayload) -> Metric: + del payload + raise NotImplementedError("test bundler only exercises submission-side bundling") class _SyncPlatform: @@ -150,10 +184,18 @@ def test_resolve_workspace_requires_explicit_or_default_workspace() -> None: http_utils.resolve_workspace(cast(NeMoPlatform, _PlatformWithoutWorkspace()), None, strict=True) -def test_metric_config_rejects_non_serializable_metric() -> None: - """Metrics must expose a JSON model dump for evaluator plugin execution.""" - with pytest.raises(TypeError, match="model_dump"): - metric_config(object()) +def test_bundle_metrics_for_spec_rejects_non_metric_object() -> None: + """Metrics must satisfy the runtime Metric protocol before plugin execution.""" + with pytest.raises(MetricBundlingError, match="Metric protocol"): + bundle_metrics_for_spec(cast(Any, object()), metric_payload_bundler=CloudpickleMetricPayloadBundler()) + + +def test_build_evaluate_spec_requires_metric_payload_bundler() -> None: + with pytest.raises(MetricPayloadBundlerPolicyError, match="CloudpickleMetricPayloadBundler"): + _build_evaluate_spec( + metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + request=EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]), + ) def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: @@ -161,6 +203,7 @@ def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: model = Model(url="https://model.test/v1", name="model-a") spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + metric_payload_bundler=CloudpickleMetricPayloadBundler(), request=EvaluationRequest( dataset=[{"expected": "a", "output": "a"}], target=model, @@ -172,10 +215,27 @@ def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: assert spec.prompt_template == "Answer: {{item.input}}" +def test_build_evaluate_spec_uses_selected_bundler_for_all_runtime_metrics() -> None: + """Submission bundles all outgoing runtime metrics with the caller-selected bundler.""" + metric_a = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + metric_b = ExactMatchMetric(reference="{{item.other_expected}}", candidate="{{item.other_output}}") + bundler = _RecordingMetricPayloadBundler() + + spec = _build_evaluate_spec( + metrics=[metric_a, metric_b], + metric_payload_bundler=bundler, + request=EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]), + ) + + assert bundler.metrics == [metric_a, metric_b] + assert [metric.metric_type for metric in spec.metrics] == ["exact-match", "exact-match"] + + def test_build_evaluate_spec_excludes_aggregate_fields() -> None: """Evaluator specs should not persist result-shaping options.""" spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + metric_payload_bundler=CloudpickleMetricPayloadBundler(), request=EvaluationRequest( dataset=[{"expected": "a", "output": "a"}], params=RunConfig(), @@ -193,6 +253,7 @@ def test_build_evaluate_spec_preserves_fileset_ref_dataset() -> None: spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + metric_payload_bundler=CloudpickleMetricPayloadBundler(), request=EvaluationRequest(dataset=cast(Any, dataset)), ) @@ -203,6 +264,7 @@ def test_build_evaluate_spec_synthesizes_fileset_ref_fragment_from_dataset_glob_ """FilesetRef datasets should encode dataset_glob_pattern as the existing fragment selector syntax.""" spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + metric_payload_bundler=CloudpickleMetricPayloadBundler(), request=EvaluationRequest( dataset=cast(Any, FilesetRef(root="default/helpsteer2")), dataset_glob_pattern="validation/*.jsonl", @@ -217,6 +279,7 @@ def test_build_evaluate_spec_rejects_fileset_ref_fragment_and_dataset_glob_patte with pytest.raises(ValueError, match=r"dataset_glob_pattern.*FilesetRef"): _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + metric_payload_bundler=CloudpickleMetricPayloadBundler(), request=EvaluationRequest( dataset=cast(Any, FilesetRef(root="default/helpsteer2#validation/*.jsonl")), dataset_glob_pattern="train/*.jsonl", @@ -277,7 +340,7 @@ def test_sync_executor_creates_evaluator_job() -> None: assert job.name == "job-123" assert job.job.status == PlatformJobStatus.CREATED assert job.job.spec is not None - assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH + assert _single_metric(job.job.spec).metric_type == "exact-match" platform._client.post.assert_called_once_with( "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs", json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON}, @@ -315,7 +378,7 @@ def test_sync_executor_create_uses_platform_workspace_by_default() -> None: job = executor.create(spec=_EXACT_MATCH_EVALUATE_SPEC) assert job.name == "job-123" assert job.job.spec is not None - assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH + assert _single_metric(job.job.spec).metric_type == "exact-match" platform._client.post.assert_called_once_with( "http://test:8000/apis/evaluator/v2/workspaces/platform-ws/evaluate/jobs", json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON}, @@ -453,6 +516,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, + metric_payload_bundler=None, ) assert job is expected_job @@ -463,6 +527,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, + metric_payload_bundler=None, ) def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: @@ -484,6 +549,7 @@ def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: target=None, dataset_glob_pattern=None, prompt_template=None, + metric_payload_bundler=None, ) @@ -564,6 +630,69 @@ def test_run_uses_local_executor_execution(self, mocker: MockerFixture) -> None: remote_evaluate.assert_not_called() +def test_sync_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None: + platform = _SyncPlatform() + executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) + expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) + sdk_evaluator = mocker.Mock() + sdk_evaluator.run_sync.return_value = expected + sdk_evaluator_cls = mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator) + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + dataset = [{"expected": "a", "output": "a"}] + + result = executor.evaluate( + metric=metric, + dataset=dataset, + params=RunConfig(parallelism=2), + ) + + assert result is expected + sdk_evaluator_cls.assert_called_once_with() + sdk_evaluator.run_sync.assert_called_once_with( + metrics=metric, + dataset=dataset, + config=RunConfig(parallelism=2), + target=None, + dataset_glob_pattern=None, + prompt_template=None, + ) + + +def test_sync_executor_evaluate_resolves_fileset_ref_before_calling_sdk(mocker: MockerFixture) -> None: + platform = _SyncPlatform() + executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) + expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) + sdk_evaluator = mocker.Mock() + sdk_evaluator.run_sync.return_value = expected + mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator) + downloaded_path = Path("/tmp/downloaded-dataset") + download_dataset_sync = mocker.patch( + "nemo_evaluator.sdk._executor.download_dataset_sync", + return_value=downloaded_path, + ) + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + dataset = FilesetRef(root="default/helpsteer2") + + result = executor.evaluate( + metric=metric, + dataset=dataset, + dataset_glob_pattern="validation/*.jsonl", + ) + + assert result is expected + download_dataset_sync.assert_called_once() + assert download_dataset_sync.call_args.kwargs["sdk"] is platform + assert download_dataset_sync.call_args.kwargs["dataset"] == FilesetRef(root="default/helpsteer2#validation/*.jsonl") + sdk_evaluator.run_sync.assert_called_once_with( + metrics=metric, + dataset=downloaded_path, + config=RunConfig(), + target=None, + dataset_glob_pattern=None, + prompt_template=None, + ) + + def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None: platform = _SyncPlatform() executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) @@ -579,23 +708,16 @@ def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: Mocke result = executor.evaluate_remote( metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), request=request, + metric_payload_bundler=CloudpickleMetricPayloadBundler(), ) assert result == expected - create.assert_called_once_with( - spec=EvaluateSpec.model_validate( - { - "metric": { - "type": "exact-match", - "reference": "{{item.expected}}", - "candidate": "{{item.output}}", - }, - "dataset": [{"expected": "a", "output": "a"}], - "params": {"limit_samples": None, "parallelism": 2}, - } - ), - workspace="platform-ws", - ) + create.assert_called_once() + assert create.call_args.kwargs["workspace"] == "platform-ws" + created_spec = create.call_args.kwargs["spec"] + assert _single_metric(created_spec).metric_type == "exact-match" + assert created_spec.dataset == [{"expected": "a", "output": "a"}] + assert created_spec.params == RunConfig(parallelism=2) job_resource.wait_until_done.assert_called_once_with( poll_interval_seconds=10.0, job_timeout_seconds=3600.0, @@ -662,7 +784,7 @@ async def test_async_executor_creates_evaluator_job(mocker: MockerFixture) -> No assert job.name == "job-123" assert job.job.status == PlatformJobStatus.CREATED assert job.job.spec is not None - assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH + assert _single_metric(job.job.spec).metric_type == "exact-match" platform._client.post.assert_awaited_once_with( "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs", json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON}, @@ -794,6 +916,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, + metric_payload_bundler=None, ) assert job is expected_job @@ -804,6 +927,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, + metric_payload_bundler=None, ) @pytest.mark.asyncio @@ -826,6 +950,7 @@ async def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: target=None, dataset_glob_pattern=None, prompt_template=None, + metric_payload_bundler=None, ) @@ -934,6 +1059,35 @@ async def test_async_executor_remote_submit_uses_platform_async_client_headers_a http_client_cls.assert_not_called() +@pytest.mark.asyncio +async def test_async_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None: + platform = _AsyncPlatform() + executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) + expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) + sdk_evaluator = mocker.Mock() + sdk_evaluator.run = AsyncMock(return_value=expected) + sdk_evaluator_cls = mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator) + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + dataset = [{"expected": "a", "output": "a"}] + + result = await executor.evaluate( + metric=metric, + dataset=dataset, + params=RunConfig(parallelism=2), + ) + + assert result is expected + sdk_evaluator_cls.assert_called_once_with() + sdk_evaluator.run.assert_awaited_once_with( + metrics=metric, + dataset=dataset, + config=RunConfig(parallelism=2), + target=None, + dataset_glob_pattern=None, + prompt_template=None, + ) + + @pytest.mark.asyncio async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None: platform = _AsyncPlatform() @@ -951,23 +1105,16 @@ async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker result = await executor.evaluate_remote( metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), request=request, + metric_payload_bundler=CloudpickleMetricPayloadBundler(), ) assert result == expected - create.assert_awaited_once_with( - spec=EvaluateSpec.model_validate( - { - "metric": { - "type": "exact-match", - "reference": "{{item.expected}}", - "candidate": "{{item.output}}", - }, - "dataset": [{"expected": "a", "output": "a"}], - "params": {"limit_samples": None, "parallelism": 2}, - } - ), - workspace="platform-ws", - ) + create.assert_awaited_once() + assert create.call_args.kwargs["workspace"] == "platform-ws" + created_spec = create.call_args.kwargs["spec"] + assert _single_metric(created_spec).metric_type == "exact-match" + assert created_spec.dataset == [{"expected": "a", "output": "a"}] + assert created_spec.params == RunConfig(parallelism=2) job_resource.wait_until_done.assert_awaited_once_with( poll_interval_seconds=10.0, job_timeout_seconds=3600.0, diff --git a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py index a52bc5b8a4..1458aff7b1 100644 --- a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py +++ b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py @@ -29,6 +29,9 @@ metric_job_status_details_value, metric_job_status_value, ) +from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler +from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -44,11 +47,12 @@ "name": "job-123", "status": "created", "spec": { - "metric": { - "type": "exact-match", - "reference": "{{item.expected}}", - "candidate": "{{item.output}}", - }, + "metrics": [ + bundle_metric( + ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + CloudpickleMetricPayloadBundler(), + ).model_dump(mode="json") + ], "dataset": [{"expected": "a", "output": "a"}], }, } diff --git a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py index da2c696e7b..85a2ff5d59 100644 --- a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py +++ b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py @@ -107,7 +107,11 @@ def test_evaluate_remote_delegates_to_resource_executor_remote_path(self, mocker result = NMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) assert result is expected - remote_evaluate.assert_called_once_with(metric=metric, request=request) + remote_evaluate.assert_called_once_with( + metric=metric, + request=request, + metric_payload_bundler=None, + ) local_evaluate.assert_not_called() def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: MockerFixture) -> None: @@ -127,7 +131,10 @@ def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: M result = NMPBackend(resource).evaluate_benchmark(metrics=metrics, request=request) assert result is expected - evaluate_benchmark.assert_called_once_with(metrics=metrics, request=request) + evaluate_benchmark.assert_called_once_with( + metrics=metrics, + request=request, + ) def test_evaluate_benchmark_remote_raises_without_local_run(self, mocker: MockerFixture) -> None: resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) @@ -206,7 +213,11 @@ async def test_evaluate_remote_delegates_to_resource_executor_remote_path(self, result = await AsyncNMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) assert result is expected - remote_evaluate.assert_awaited_once_with(metric=metric, request=request) + remote_evaluate.assert_awaited_once_with( + metric=metric, + request=request, + metric_payload_bundler=None, + ) local_evaluate.assert_not_awaited() @pytest.mark.asyncio @@ -227,7 +238,10 @@ async def test_evaluate_benchmark_local_delegates_to_resource_executor(self, moc result = await AsyncNMPBackend(resource).evaluate_benchmark(metrics=metrics, request=request) assert result is expected - evaluate_benchmark.assert_awaited_once_with(metrics=metrics, request=request) + evaluate_benchmark.assert_awaited_once_with( + metrics=metrics, + request=request, + ) @pytest.mark.asyncio async def test_evaluate_benchmark_remote_raises_without_local_run(self, mocker: MockerFixture) -> None: diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl index 8405ae9fab..75d18f0790 100644 --- a/third_party/licenses.jsonl +++ b/third_party/licenses.jsonl @@ -44,6 +44,7 @@ {"name": "click-plugins", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "click-repl", "license": "MIT", "compatible": true} {"name": "clickhouse-connect", "license": "APACHE-2.0", "compatible": true} +{"name": "cloudpickle", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "colorama", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "colorlog", "license": "MIT", "compatible": true} {"name": "cryptography", "license": "APACHE-2.0", "compatible": true} diff --git a/uv.lock b/uv.lock index 7394370c32..920db4a790 100644 --- a/uv.lock +++ b/uv.lock @@ -4911,6 +4911,7 @@ name = "nemo-evaluator-plugin" version = "0.1.0" source = { editable = "plugins/nemo-evaluator" } dependencies = [ + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nemo-evaluator-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nemo-platform", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, @@ -4928,6 +4929,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cloudpickle", specifier = ">=3.1.1" }, { name = "nemo-evaluator-sdk", editable = "packages/nemo_evaluator_sdk" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, @@ -5071,6 +5073,7 @@ all = [ { name = "botocore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "celery", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "clickhouse-connect", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer-nemo", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "dataclasses-json", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, @@ -5429,6 +5432,7 @@ nemo-data-designer-plugin = [ { name = "pandas", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, ] nemo-evaluator-plugin = [ + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nemo-evaluator-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, @@ -5510,6 +5514,7 @@ plugins = [ { name = "anthropic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "boto3", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "botocore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer-nemo", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "dataclasses-json", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, @@ -5571,6 +5576,7 @@ services = [ { name = "botocore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "celery", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "clickhouse-connect", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "data-designer-nemo", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, { name = "dataclasses-json", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" }, @@ -5739,6 +5745,10 @@ requires-dist = [ { name = "clickhouse-connect", marker = "extra == 'all'", specifier = ">=0.7,<1.0" }, { name = "clickhouse-connect", marker = "extra == 'intake-service'", specifier = ">=0.7,<1.0" }, { name = "clickhouse-connect", marker = "extra == 'services'", specifier = ">=0.7,<1.0" }, + { name = "cloudpickle", marker = "extra == 'all'", specifier = ">=3.1.1" }, + { name = "cloudpickle", marker = "extra == 'nemo-evaluator-plugin'", specifier = ">=3.1.1" }, + { name = "cloudpickle", marker = "extra == 'plugins'", specifier = ">=3.1.1" }, + { name = "cloudpickle", marker = "extra == 'services'", specifier = ">=3.1.1" }, { name = "data-designer", marker = "extra == 'all'", specifier = "==0.6.0" }, { name = "data-designer", marker = "extra == 'data-designer-nemo'", specifier = "==0.6.0" }, { name = "data-designer", marker = "extra == 'nemo-anonymizer-plugin'", specifier = "==0.6.0" }, From 4e5b80f0fbbca6e1f846381b7c9d6d1f42ccd3d3 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 1 Jun 2026 12:00:43 -0300 Subject: [PATCH 2/2] fix(evaluator): address metric bundle review feedback Signed-off-by: Sandy Chapman --- .../examples/plugin_examples.py | 44 +++++- .../src/nemo_evaluator/jobs/compiler.py | 41 +----- .../src/nemo_evaluator/sdk/_executor.py | 48 +++---- .../src/nemo_evaluator/sdk/resources.py | 20 ++- .../sdk/standalone_sdk/backend.py | 16 ++- .../shared/metric_bundles/bundles.py | 47 ++++--- .../shared/metric_bundles/cloudpickle.py | 61 ++++++-- .../shared/metric_bundles/test_cloudpickle.py | 130 ++++++++++++++---- .../nemo-evaluator/tests/test_evaluate_job.py | 61 +++++--- plugins/nemo-evaluator/tests/test_sdk.py | 101 +++++++++----- .../tests/test_sdk_job_resources.py | 4 +- .../tests/test_standalone_sdk_backend.py | 42 +++++- 12 files changed, 429 insertions(+), 186 deletions(-) diff --git a/packages/nemo_evaluator_sdk/examples/plugin_examples.py b/packages/nemo_evaluator_sdk/examples/plugin_examples.py index 0d8aea80e8..fa40366161 100644 --- a/packages/nemo_evaluator_sdk/examples/plugin_examples.py +++ b/packages/nemo_evaluator_sdk/examples/plugin_examples.py @@ -15,6 +15,7 @@ from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Any, cast +from nemo_evaluator.jobs.evaluate import EvaluateSpec from nemo_evaluator.sdk.resources import AsyncEvaluator from nemo_evaluator.sdk.resources import Evaluator as SyncEvaluator from nemo_evaluator.sdk.types import ( @@ -23,10 +24,12 @@ RunConfig, RunConfigOnlineModel, ) +from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator_sdk.enums import MetricType from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric -from nemo_evaluator_sdk.metrics.protocol import Metric +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import ( InferenceParams, JSONScoreParser, @@ -74,6 +77,23 @@ ) +class CustomResponseLengthMetric: + """Tiny custom metric used to demonstrate code-generated metric bundles.""" + + type = "custom-response-length" + description = "Scores each row by response length." + labels = {"source": "plugin-example"} + + def output_spec(self) -> list[MetricOutputSpec]: + """Return the metric outputs recorded in the bundle metadata.""" + return [MetricOutputSpec.continuous_score("response-length")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: + """Score one row with a deterministic custom Python implementation.""" + response = str(input.row.data.get("response", "")) + return MetricResult(outputs=[MetricOutput(name="response-length", value=float(len(response)))]) + + def configure_example_logging() -> None: """Enable SDK progress logs when this example file is executed directly.""" logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") @@ -306,6 +326,26 @@ def _online_exact_match_metric() -> ExactMatchMetric: return ExactMatchMetric(type=MetricType.EXACT_MATCH, reference="{{item.response}}") +def build_custom_metric_submit_spec_example() -> dict[str, Any]: + """Return the generated job spec for remote custom metric submission. + + The cloudpickle payload contains base64-encoded Python bytes. It is not a + field users should hand-author; generate it from the metric object with a + metric payload packager, or pass the packager to ``submit``. + """ + metric = CustomResponseLengthMetric() + spec = EvaluateSpec.model_validate( + { + "metrics": [ + bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json"), + ], + "dataset": [{"response": "Paris is the capital of France."}], + "params": RunConfig(limit_samples=1).model_dump(mode="json"), + } + ) + return spec.model_dump(mode="json") + + def _assert_exact_match_result(result: EvaluationResult, *, workflow: str, expected_rows: int) -> None: """Assert the deterministic offline exact-match examples scored every selected row.""" if len(result.row_scores) != expected_rows: @@ -345,6 +385,7 @@ async def _evaluate_metric( metric=metric, dataset=dataset, config=config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), **run_kwargs, ) print(f"Submitted evaluator plugin job: {job.name}") @@ -506,6 +547,7 @@ def run_nmp_online_metric_example_sync_client( metric=metric, dataset=dataset, config=config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), **run_kwargs, ) print(f"Submitted evaluator plugin job: {job.name}") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py index bd7a712f21..14dc26219e 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py @@ -17,24 +17,18 @@ ) from nemo_platform_plugin.jobs.constants import ( DEFAULT_JOB_STORAGE_PATH, - EPHEMERAL_TASK_STORAGE_PATH_ENVVAR, PERSISTENT_JOB_STORAGE_PATH_ENVVAR, ) from nmp.common.jobs.image import get_qualified_image -from nmp.evaluator.app.values import FilesetRef -DATASET_DOWNLOAD_STEP_NAME = "dataset-download" EVALUATE_STEP_NAME = "evaluate" +_RESERVED_SECRET_ENV_NAMES = frozenset({PERSISTENT_JOB_STORAGE_PATH_ENVVAR}) def compile_evaluate_job(spec: EvaluateSpec, *, profile: str | None = None) -> PlatformJobSpec: """Compile a bundle-native evaluator plugin job.""" _validate_evaluate_spec(spec) - steps: list[PlatformJobStep] = [] - if isinstance(spec.dataset, FilesetRef): - steps.append(_fileset_download_step(spec.dataset)) - steps.append(_evaluate_step(spec, profile)) - return PlatformJobSpec(steps=steps) + return PlatformJobSpec(steps=[_evaluate_step(spec, profile)]) def _validate_evaluate_spec(spec: EvaluateSpec) -> None: @@ -52,36 +46,9 @@ def _validate_evaluate_spec(spec: EvaluateSpec) -> None: raise TypeError("offline evaluation requires RunConfig") -def _fileset_download_step(dataset: FilesetRef) -> PlatformJobStep: - scratch_path = "${" + EPHEMERAL_TASK_STORAGE_PATH_ENVVAR + "}" - target_download_dir = "${" + PERSISTENT_JOB_STORAGE_PATH_ENVVAR + "}/datasets" - return PlatformJobStep( - name=DATASET_DOWNLOAD_STEP_NAME, - executor=CPUExecutionProviderSpec( - provider="cpu", - container=ContainerSpec( - image=get_qualified_image("nmp-cpu-tasks"), - entrypoint=["python", "-m", "nmp.evaluator.tasks.download_fileset"], - command=[ - "--local-dir", - scratch_path, - "--target-dir", - target_download_dir, - "--dataset", - dataset.model_dump_json(), - ], - ), - ), - environment=[ - EnvironmentVariable( - name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR, - value=DEFAULT_JOB_STORAGE_PATH, - ) - ], - ) - - def _add_secret_ref(secret_refs: dict[str, str], env_name: str, secret_name: str) -> None: + if env_name in _RESERVED_SECRET_ENV_NAMES: + raise ValueError(f"{env_name!r} is reserved and cannot be sourced from secret refs") existing = secret_refs.get(env_name) if existing is not None and existing != secret_name: raise ValueError(f"conflicting secret references for environment variable {env_name!r}") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index 63855855a2..ff192a29ba 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -23,7 +23,7 @@ ) from nemo_evaluator.sdk.types import PluginDatasetInput from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result -from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, MetricPayloadBundler, bundle_metric +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, MetricBundlePackager, bundle_metric from nemo_evaluator_sdk import Evaluator as SDKEvaluator from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows from nemo_evaluator_sdk.execution.config import EvaluationRequest, normalize_params @@ -50,17 +50,17 @@ _ResolvedDataset = DatasetInput | str | Path -class MetricPayloadBundlerPolicyError(RuntimeError): - """Raised when plugin backend metric bundling is not configured.""" +class MetricBundlePackagerPolicyError(RuntimeError): + """Raised when plugin backend metric packaging is not configured.""" -def _require_metric_payload_bundler(metric_payload_bundler: MetricPayloadBundler | None) -> MetricPayloadBundler: - if metric_payload_bundler is None: - raise MetricPayloadBundlerPolicyError( - "Bundling runtime metrics for evaluator plugin submission requires an explicit metric_payload_bundler. " - "Pass CloudpickleMetricPayloadBundler() to opt in to cloudpickle metric bundles." +def _require_metric_bundle_packager(metric_bundle_packager: MetricBundlePackager | None) -> MetricBundlePackager: + if metric_bundle_packager is None: + raise MetricBundlePackagerPolicyError( + "Packaging runtime metrics for evaluator plugin submission requires an explicit metric_bundle_packager. " + "Pass CloudpickleMetricBundlePackager() to opt in to cloudpickle metric bundles." ) - return metric_payload_bundler + return metric_bundle_packager def _dataset_config(request: EvaluationRequest) -> list[dict[str, Any]] | FilesetRef: @@ -128,12 +128,12 @@ def _build_evaluate_spec( *, metrics: Metric | Sequence[Metric], request: EvaluationRequest, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluateSpec: """Build the evaluator plugin spec shared by local and remote execution.""" - effective_bundler = _require_metric_payload_bundler(metric_payload_bundler) + effective_packager = _require_metric_bundle_packager(metric_bundle_packager) spec = { - "metrics": bundle_metrics_for_spec(metrics, metric_payload_bundler=effective_bundler), + "metrics": bundle_metrics_for_spec(metrics, metric_bundle_packager=effective_packager), "dataset": _dataset_config(request), "params": request.params.model_dump(mode="json") if request.params else None, } @@ -215,13 +215,13 @@ def evaluate_remote( *, metric: Metric, request: EvaluationRequest, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluationResult: """Submit, poll, and download a remote evaluator plugin metric job.""" spec = _build_evaluate_spec( metrics=metric, request=request, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) job = self.create( @@ -275,7 +275,7 @@ def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluatorJobResource: """Submit a remote evaluator plugin metric job and return the job resource.""" request = EvaluationRequest( @@ -288,7 +288,7 @@ def submit( spec = _build_evaluate_spec( metrics=metric, request=request, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) job = self.create( @@ -394,7 +394,7 @@ async def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> AsyncEvaluatorJobResource: """Submit a remote evaluator plugin metric job and return the job resource.""" request = EvaluationRequest( @@ -407,7 +407,7 @@ async def submit( spec = _build_evaluate_spec( metrics=metric, request=request, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) job = await self.create( @@ -421,13 +421,13 @@ async def evaluate_remote( *, metric: Metric, request: EvaluationRequest, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluationResult: """Submit, poll, and download a remote evaluator plugin metric job.""" spec = _build_evaluate_spec( metrics=metric, request=request, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) job = await self.create( @@ -492,10 +492,10 @@ async def evaluate_benchmark( def bundle_metrics_for_spec( - metrics: Metric | Sequence[Metric], *, metric_payload_bundler: MetricPayloadBundler + metrics: Metric | Sequence[Metric], *, metric_bundle_packager: MetricBundlePackager ) -> list[MetricBundle]: - """Bundle one metric or a benchmark metric sequence for an evaluator plugin spec.""" + """Package one metric or a benchmark metric sequence for an evaluator plugin spec.""" if isinstance(metrics, Sequence) and not isinstance(metrics, (str, bytes)): metric_sequence = cast(Sequence[Metric], metrics) - return [bundle_metric(metric, metric_payload_bundler) for metric in metric_sequence] - return [bundle_metric(cast(Metric, metrics), metric_payload_bundler)] + return [bundle_metric(metric, metric_bundle_packager) for metric in metric_sequence] + return [bundle_metric(cast(Metric, metrics), metric_bundle_packager)] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index c5a129eea8..2acdb7bad8 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -24,7 +24,7 @@ RunConfigOnline, RunConfigOnlineModel, ) -from nemo_evaluator.shared.metric_bundles.bundles import MetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundlePackager from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, @@ -85,9 +85,14 @@ def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluatorJobResource: """Submit a metric job through the evaluator plugin executor.""" + if metric_bundle_packager is None: + raise ValueError( + "metric_bundle_packager is required for submit(); " + "pass CloudpickleMetricBundlePackager() to enable metric bundling." + ) return self._executor.submit( metric=metric, dataset=dataset, @@ -95,7 +100,7 @@ def submit( target=target, dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) def run( @@ -192,9 +197,14 @@ async def submit( target: Model | Agent | None = None, dataset_glob_pattern: str | None = None, prompt_template: str | dict[str, Any] | None = None, - metric_payload_bundler: MetricPayloadBundler | None = None, + metric_bundle_packager: MetricBundlePackager | None = None, ) -> AsyncEvaluatorJobResource: """Submit a metric job through the evaluator plugin executor.""" + if metric_bundle_packager is None: + raise ValueError( + "metric_bundle_packager is required for submit(); " + "pass CloudpickleMetricBundlePackager() to enable metric bundling." + ) return await self._executor.submit( metric=metric, dataset=dataset, @@ -202,7 +212,7 @@ async def submit( target=target, dataset_glob_pattern=dataset_glob_pattern, prompt_template=prompt_template, - metric_payload_bundler=metric_payload_bundler, + metric_bundle_packager=metric_bundle_packager, ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py index 23e314ac13..7f49c6b255 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py @@ -10,7 +10,7 @@ from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.sdk.types import ExecutionMode -from nemo_evaluator.shared.metric_bundles.bundles import MetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundlePackager from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult @@ -23,6 +23,12 @@ def _reject_unsupported_hooks(request: EvaluationRequest) -> None: raise NotImplementedError("preprocess_hooks and postprocess_hooks are not supported.") +def _require_remote_packager(metric_bundle_packager: MetricBundlePackager | None) -> MetricBundlePackager: + if metric_bundle_packager is None: + raise ValueError("metric_bundle_packager is required when execution_mode='remote'.") + return metric_bundle_packager + + @dataclass(frozen=True, slots=True) class NMPBackend: """Sync standalone evaluator SDK backend backed by a plugin evaluator resource. @@ -31,7 +37,7 @@ class NMPBackend: resource: Evaluator execution_mode: ExecutionMode = "local" - metric_payload_bundler: MetricPayloadBundler | None = None + metric_bundle_packager: MetricBundlePackager | None = None def evaluate( self, @@ -45,7 +51,7 @@ def evaluate( return self.resource._executor.evaluate_remote( metric=metric, request=request, - metric_payload_bundler=self.metric_payload_bundler, + metric_bundle_packager=_require_remote_packager(self.metric_bundle_packager), ) return self.resource._executor.evaluate( metric=metric, @@ -81,7 +87,7 @@ class AsyncNMPBackend: resource: AsyncEvaluator execution_mode: ExecutionMode = "local" - metric_payload_bundler: MetricPayloadBundler | None = None + metric_bundle_packager: MetricBundlePackager | None = None async def evaluate( self, @@ -95,7 +101,7 @@ async def evaluate( return await self.resource._executor.evaluate_remote( metric=metric, request=request, - metric_payload_bundler=self.metric_payload_bundler, + metric_bundle_packager=_require_remote_packager(self.metric_bundle_packager), ) return await self.resource._executor.evaluate( metric=metric, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py index 6ed6c6c0b5..bef57ebe50 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/bundles.py @@ -68,7 +68,7 @@ class MetricBundlePayload(BaseModel, ABC): @property @abstractmethod def kind(self) -> str: - """Payload discriminator used to select the bundler implementation.""" + """Payload discriminator used to select the packager implementation.""" ... @property @@ -78,14 +78,14 @@ def digest(self) -> str: ... -class MetricPayloadBundler(Protocol): - """Interface for metric bundle payload implementations.""" +class MetricBundlePackager(Protocol): + """Strategy for packaging a runtime metric into a bundle payload and loading it later.""" - def bundle(self, metric: Metric) -> MetricBundlePayload: - """Serialize a runtime metric object to a format-specific payload.""" + def package(self, metric: Metric) -> MetricBundlePayload: + """Package a runtime metric object into a format-specific payload.""" ... - def unbundle(self, payload: MetricBundlePayload) -> Metric: + def load(self, payload: MetricBundlePayload) -> Metric: """Hydrate an executable metric from a bundle payload.""" ... @@ -93,7 +93,7 @@ def unbundle(self, payload: MetricBundlePayload) -> Metric: @dataclass(frozen=True) class _MetricBundleRegistration: payload_type: type[MetricBundlePayload] - payload_bundler_factory: Callable[[], MetricPayloadBundler] + packager_factory: Callable[[], MetricBundlePackager] _BUNDLE_REGISTRY: dict[str, _MetricBundleRegistration] = {} @@ -110,15 +110,21 @@ def register_metric_bundle_kind( kind: str, *, payload_type: type[MetricBundlePayload], - payload_bundler_factory: Callable[[], MetricPayloadBundler], + packager_factory: Callable[[], MetricBundlePackager], ) -> None: - """Register the payload model and payload bundler factory for a bundle kind.""" + """Register the payload model and packager factory for a bundle kind.""" if not kind: raise ValueError("metric bundle payload kind must not be empty") - _BUNDLE_REGISTRY[kind] = _MetricBundleRegistration( + registration = _MetricBundleRegistration( payload_type=payload_type, - payload_bundler_factory=payload_bundler_factory, + packager_factory=packager_factory, ) + existing = _BUNDLE_REGISTRY.get(kind) + if existing is not None: + if existing == registration: + return + raise ValueError(f"metric bundle payload kind already registered: {kind}") + _BUNDLE_REGISTRY[kind] = registration class MetricBundle(BaseModel): @@ -168,20 +174,20 @@ def _output_names_must_be_unique(self) -> MetricBundle: return self -def metric_payload_bundler_for_payload(payload: MetricBundlePayload) -> MetricPayloadBundler: - """Create the payload bundler registered for a metric bundle payload.""" +def metric_bundle_packager_for_payload(payload: MetricBundlePayload) -> MetricBundlePackager: + """Create the packager registered for a metric bundle payload.""" kind = _payload_kind(payload) registration = _BUNDLE_REGISTRY.get(kind) if registration is None: raise MetricBundlingError(f"unsupported metric bundle payload kind: {kind}") - return registration.payload_bundler_factory() + return registration.packager_factory() -def bundle_metric(metric: Metric, bundler: MetricPayloadBundler) -> MetricBundle: +def bundle_metric(metric: Metric, packager: MetricBundlePackager) -> MetricBundle: """Build a standard metric bundle envelope around a format-specific payload.""" if not isinstance(metric, Metric): raise MetricBundlingError("object does not satisfy the Metric protocol") - payload = bundler.bundle(metric) + payload = packager.package(metric) return MetricBundle( metric_type=validate_metric_type(metric), metadata=metric_metadata(metric), @@ -193,8 +199,8 @@ def bundle_metric(metric: Metric, bundler: MetricPayloadBundler) -> MetricBundle def unbundle_metric(bundle: MetricBundle) -> Metric: """Hydrate a runtime metric from a standard metric bundle envelope.""" - payload_bundler = metric_payload_bundler_for_payload(bundle.payload) - hydrated_metric = payload_bundler.unbundle(bundle.payload) + packager = metric_bundle_packager_for_payload(bundle.payload) + hydrated_metric = packager.load(bundle.payload) _validate_metric_matches_bundle(hydrated_metric, bundle) return hydrated_metric @@ -204,9 +210,8 @@ def _validate_metric_matches_bundle(metric: object, bundle: MetricBundle) -> Non if not isinstance(metric, Metric): raise MetricBundlingError("unbundled object does not satisfy the Metric protocol") - output_names = [output.name for output in metric.output_spec()] - bundled_output_names = [output.name for output in bundle.outputs] - if output_names != bundled_output_names: + hydrated_outputs = [BundledMetricOutputSpec.from_output_spec(output) for output in metric.output_spec()] + if hydrated_outputs != bundle.outputs: raise MetricBundlingError("unbundled metric output spec does not match bundle metadata") if validate_metric_type(metric) != bundle.metric_type: raise MetricBundlingError("unbundled metric type does not match bundle metadata") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py index af616b64e6..f557d0e4f9 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/shared/metric_bundles/cloudpickle.py @@ -8,19 +8,53 @@ import hashlib import pickle import platform +import sys from typing import Annotated, Literal import cloudpickle from nemo_evaluator.shared.metric_bundles.bundles import ( + MetricBundlePackager, MetricBundlePayload, MetricBundlingError, - MetricPayloadBundler, register_metric_bundle_kind, ) from nemo_evaluator_sdk.metrics.protocol import Metric -from pydantic import ConfigDict, Field, computed_field +from pydantic import ConfigDict, Field, computed_field, field_validator -NonEmptyBytes = Annotated[bytes, Field(min_length=1)] +MAX_CLOUDPICKLE_PAYLOAD_BYTES = 10 * 1024 * 1024 +CloudpickleBlob = Annotated[bytes, Field(min_length=1, max_length=MAX_CLOUDPICKLE_PAYLOAD_BYTES)] + + +def _format_bytes(value: int) -> str: + return f"{value / (1024 * 1024):.1f} MiB" + + +def _validate_payload_size(blob: bytes) -> bytes: + if len(blob) > MAX_CLOUDPICKLE_PAYLOAD_BYTES: + raise MetricBundlingError( + "cloudpickle metric payload is " + f"{_format_bytes(len(blob))}; maximum allowed is {_format_bytes(MAX_CLOUDPICKLE_PAYLOAD_BYTES)}" + ) + return blob + + +def _python_major_minor(version: str) -> tuple[int, int]: + try: + major, minor, *_ = version.split(".") + return int(major), int(minor) + except ValueError as exc: + raise MetricBundlingError(f"invalid cloudpickle payload python_version: {version!r}") from exc + + +def _validate_python_version(payload: CloudpickleMetricPayload) -> None: + payload_version = _python_major_minor(payload.python_version) + runtime_version = sys.version_info[:2] + if payload_version != runtime_version: + raise MetricBundlingError( + "cloudpickle metric payload was created with " + f"Python {payload.python_version}, but this runtime is Python {platform.python_version()}; " + "recreate the metric bundle with the runtime Python version." + ) class CloudpickleMetricPayload(MetricBundlePayload): @@ -31,7 +65,12 @@ class CloudpickleMetricPayload(MetricBundlePayload): python_version: str cloudpickle_version: str pickle_protocol: int - blob: NonEmptyBytes + blob: CloudpickleBlob + + @field_validator("blob") + @classmethod + def _blob_must_fit_step_config(cls, blob: bytes) -> bytes: + return _validate_payload_size(blob) @property def kind(self) -> Literal["cloudpickle"]: @@ -47,6 +86,7 @@ def digest(self) -> str: @classmethod def from_blob(cls, blob: bytes) -> CloudpickleMetricPayload: """Create a JSON-safe cloudpickle payload from raw bytes.""" + _validate_payload_size(blob) return cls( python_version=platform.python_version(), cloudpickle_version=cloudpickle.__version__, @@ -55,24 +95,25 @@ def from_blob(cls, blob: bytes) -> CloudpickleMetricPayload: ) -class CloudpickleMetricPayloadBundler(MetricPayloadBundler): - """Cloudpickle-backed metric payload bundler. +class CloudpickleMetricBundlePackager(MetricBundlePackager): + """Cloudpickle-backed metric bundle packager. Cloudpickle bundles execute arbitrary Python code when hydrated. This implementation is intended for explicit opt-in development/MVP use. """ - def bundle(self, metric: Metric) -> MetricBundlePayload: - """Serialize a runtime metric object to a cloudpickle payload.""" + def package(self, metric: Metric) -> MetricBundlePayload: + """Package a runtime metric object as a cloudpickle payload.""" if not isinstance(metric, Metric): raise MetricBundlingError("object does not satisfy the Metric protocol") blob = cloudpickle.dumps(metric, protocol=pickle.HIGHEST_PROTOCOL) return CloudpickleMetricPayload.from_blob(blob) - def unbundle(self, payload: MetricBundlePayload) -> Metric: + def load(self, payload: MetricBundlePayload) -> Metric: """Hydrate a metric from a cloudpickle payload.""" cloudpickle_payload = CloudpickleMetricPayload.model_validate(payload.model_dump(mode="python")) + _validate_python_version(cloudpickle_payload) hydrated_metric = cloudpickle.loads(cloudpickle_payload.blob) if not isinstance(hydrated_metric, Metric): raise MetricBundlingError("unbundled object does not satisfy the Metric protocol") @@ -82,5 +123,5 @@ def unbundle(self, payload: MetricBundlePayload) -> Metric: register_metric_bundle_kind( "cloudpickle", payload_type=CloudpickleMetricPayload, - payload_bundler_factory=CloudpickleMetricPayloadBundler, + packager_factory=CloudpickleMetricBundlePackager, ) diff --git a/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py b/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py index 9449440feb..435ae25c16 100644 --- a/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py +++ b/plugins/nemo-evaluator/tests/shared/metric_bundles/test_cloudpickle.py @@ -5,16 +5,23 @@ import hashlib from collections.abc import Sequence -from typing import cast +from typing import Literal, cast import pytest from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, + MetricBundlePackager, + MetricBundlePayload, MetricBundlingError, bundle_metric, + register_metric_bundle_kind, unbundle_metric, ) -from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayload, CloudpickleMetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.cloudpickle import ( + MAX_CLOUDPICKLE_PAYLOAD_BYTES, + CloudpickleMetricBundlePackager, + CloudpickleMetricPayload, +) from nemo_evaluator_sdk.enums import ModelFormat from nemo_evaluator_sdk.metrics.bleu import BLEUMetric from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric @@ -67,6 +74,36 @@ class _NotMetric: pass +class _TestPayload(MetricBundlePayload): + @property + def kind(self) -> Literal["test-cloudpickle-registration"]: + return "test-cloudpickle-registration" + + @property + def digest(self) -> str: + return "test-digest" + + +class _ConflictingPayload(MetricBundlePayload): + @property + def kind(self) -> Literal["test-cloudpickle-registration"]: + return "test-cloudpickle-registration" + + @property + def digest(self) -> str: + return "test-conflicting-digest" + + +class _TestPackager(MetricBundlePackager): + def package(self, metric: Metric) -> MetricBundlePayload: + del metric + return _TestPayload() + + def load(self, payload: MetricBundlePayload) -> Metric: + del payload + return _CustomMetric() + + class _EmptyTypeMetric(_CustomMetric): type = "" @@ -150,11 +187,11 @@ def _builtin_metric_cases() -> Sequence[tuple[str, Metric]]: ] -def test_cloudpickle_bundler_round_trips_builtin_metric() -> None: +def test_cloudpickle_packager_round_trips_builtin_metric() -> None: metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - bundler = CloudpickleMetricPayloadBundler() + packager = CloudpickleMetricBundlePackager() - bundle = bundle_metric(metric, bundler) + bundle = bundle_metric(metric, packager) hydrated = unbundle_metric(bundle) assert bundle.metric_type == "exact-match" @@ -165,10 +202,10 @@ def test_cloudpickle_bundler_round_trips_builtin_metric() -> None: @pytest.mark.parametrize( ("case_name", "metric"), _builtin_metric_cases(), ids=[case[0] for case in _builtin_metric_cases()] ) -def test_cloudpickle_bundler_round_trips_every_builtin_metric(case_name: str, metric: Metric) -> None: - bundler = CloudpickleMetricPayloadBundler() +def test_cloudpickle_packager_round_trips_every_builtin_metric(case_name: str, metric: Metric) -> None: + packager = CloudpickleMetricBundlePackager() - bundle = bundle_metric(metric, bundler) + bundle = bundle_metric(metric, packager) restored = MetricBundle.model_validate_json(bundle.model_dump_json()) hydrated = unbundle_metric(restored) @@ -178,10 +215,10 @@ def test_cloudpickle_bundler_round_trips_every_builtin_metric(case_name: str, me assert type(hydrated) is type(metric), case_name -def test_cloudpickle_bundler_round_trips_custom_protocol_metric() -> None: - bundler = CloudpickleMetricPayloadBundler() +def test_cloudpickle_packager_round_trips_custom_protocol_metric() -> None: + packager = CloudpickleMetricBundlePackager() - bundle = bundle_metric(_CustomMetric(), bundler) + bundle = bundle_metric(_CustomMetric(), packager) serialized = bundle.model_dump_json() restored = MetricBundle.model_validate_json(serialized) hydrated = unbundle_metric(restored) @@ -194,7 +231,7 @@ def test_cloudpickle_bundler_round_trips_custom_protocol_metric() -> None: assert isinstance(hydrated, _CustomMetric) -def test_cloudpickle_bundler_captures_metric_secrets() -> None: +def test_cloudpickle_packager_captures_metric_secrets() -> None: metric = LLMJudgeMetric( model=Model( url="https://judge.example.test/v1/chat/completions", @@ -212,14 +249,14 @@ def test_cloudpickle_bundler_captures_metric_secrets() -> None: ], ) - bundle = bundle_metric(metric, CloudpickleMetricPayloadBundler()) + bundle = bundle_metric(metric, CloudpickleMetricBundlePackager()) restored = MetricBundle.model_validate_json(bundle.model_dump_json()) assert restored.secrets == {"judge_secret": SecretRef(root="judge-secret")} -def test_cloudpickle_bundler_captures_digest_and_payload_metadata() -> None: - bundle = bundle_metric(_CustomMetric(), CloudpickleMetricPayloadBundler()) +def test_cloudpickle_packager_captures_digest_and_payload_metadata() -> None: + bundle = bundle_metric(_CustomMetric(), CloudpickleMetricBundlePackager()) payload = CloudpickleMetricPayload.model_validate(bundle.payload) serialized_payload = cast(dict[str, object], bundle.model_dump(mode="json")["payload"]) @@ -233,20 +270,67 @@ def test_cloudpickle_bundler_captures_digest_and_payload_metadata() -> None: assert bundle.outputs[0].value_json_schema["title"] == "ContinuousScore" -def test_cloudpickle_bundler_rejects_non_metric_object() -> None: +def test_cloudpickle_packager_rejects_oversized_payload() -> None: + oversized_blob = b"x" * (MAX_CLOUDPICKLE_PAYLOAD_BYTES + 1) + + with pytest.raises(MetricBundlingError, match="maximum allowed"): + CloudpickleMetricPayload.from_blob(oversized_blob) + + +def test_cloudpickle_packager_rejects_python_version_mismatch() -> None: + bundle = bundle_metric(_CustomMetric(), CloudpickleMetricBundlePackager()) + payload = CloudpickleMetricPayload.model_validate(bundle.payload) + incompatible_payload = payload.model_copy(update={"python_version": "0.0.0"}) + incompatible_bundle = bundle.model_copy(update={"payload": incompatible_payload}) + + with pytest.raises(MetricBundlingError, match="created with Python 0.0.0"): + unbundle_metric(incompatible_bundle) + + +def test_unbundle_metric_rejects_output_contract_mismatch() -> None: + bundle = bundle_metric(_CustomMetric(), CloudpickleMetricBundlePackager()) + incompatible_bundle = bundle.model_copy( + update={"outputs": [bundle.outputs[0].model_copy(update={"description": "changed"})]} + ) + + with pytest.raises(MetricBundlingError, match="output spec"): + unbundle_metric(incompatible_bundle) + + +def test_register_metric_bundle_kind_rejects_conflicting_registration() -> None: + register_metric_bundle_kind( + "test-cloudpickle-registration", + payload_type=_TestPayload, + packager_factory=_TestPackager, + ) + register_metric_bundle_kind( + "test-cloudpickle-registration", + payload_type=_TestPayload, + packager_factory=_TestPackager, + ) + + with pytest.raises(ValueError, match="already registered"): + register_metric_bundle_kind( + "test-cloudpickle-registration", + payload_type=_ConflictingPayload, + packager_factory=_TestPackager, + ) + + +def test_cloudpickle_packager_rejects_non_metric_object() -> None: with pytest.raises(MetricBundlingError, match="Metric protocol"): - bundle_metric(cast(Metric, _NotMetric()), CloudpickleMetricPayloadBundler()) + bundle_metric(cast(Metric, _NotMetric()), CloudpickleMetricBundlePackager()) -def test_cloudpickle_bundler_rejects_empty_metric_type() -> None: +def test_cloudpickle_packager_rejects_empty_metric_type() -> None: with pytest.raises(MetricBundlingError, match="metric type must not be empty"): - bundle_metric(_EmptyTypeMetric(), CloudpickleMetricPayloadBundler()) + bundle_metric(_EmptyTypeMetric(), CloudpickleMetricBundlePackager()) -def test_cloudpickle_bundler_hydrates_from_payload_without_bundle_envelope() -> None: - bundler = CloudpickleMetricPayloadBundler() - bundle = bundle_metric(_CustomMetric(), bundler) +def test_cloudpickle_packager_hydrates_from_payload_without_bundle_envelope() -> None: + packager = CloudpickleMetricBundlePackager() + bundle = bundle_metric(_CustomMetric(), packager) - hydrated = bundler.unbundle(bundle.payload) + hydrated = packager.load(bundle.payload) assert isinstance(hydrated, _CustomMetric) diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index 3cf54c5152..0faf03737b 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -24,13 +24,13 @@ from nemo_evaluator.resolvers import PlatformModelResolver, _parse_required_workspace_name from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, + MetricBundlePackager, MetricBundlePayload, - MetricPayloadBundler, bundle_metric, register_metric_bundle_kind, unbundle_metric, ) -from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator.tasks.evaluate import main as evaluate_task_main from nemo_evaluator_sdk.enums import AgentFormat from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric @@ -53,6 +53,7 @@ from nemo_platform_plugin.commands import add_job_commands from nemo_platform_plugin.job_context import JobContext, StoragePaths from nemo_platform_plugin.job_results import LocalJobResults +from nemo_platform_plugin.jobs.constants import PERSISTENT_JOB_STORAGE_PATH_ENVVAR from nemo_platform_plugin.scheduler import NemoJobScheduler from nmp.evaluator.app.values import FilesetRef from pydantic import BaseModel, ConfigDict @@ -74,7 +75,7 @@ def _exact_match_spec() -> dict: def _bundle_payload(metric) -> dict[str, Any]: - return bundle_metric(metric, CloudpickleMetricPayloadBundler()).model_dump(mode="json") + return bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json") def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None: @@ -167,12 +168,12 @@ def digest(self) -> str: return "test-strict-digest" -class _StaticMetricPayloadBundler(MetricPayloadBundler): - def bundle(self, metric: Metric) -> MetricBundlePayload: +class _StaticMetricBundlePackager(MetricBundlePackager): + def package(self, metric: Metric) -> MetricBundlePayload: del metric return _StaticMetricPayload() - def unbundle(self, payload: MetricBundlePayload) -> Metric: + def load(self, payload: MetricBundlePayload) -> Metric: del payload return _StaticMetric("test-static") @@ -180,12 +181,12 @@ def unbundle(self, payload: MetricBundlePayload) -> Metric: register_metric_bundle_kind( "test-static", payload_type=_StaticMetricPayload, - payload_bundler_factory=_StaticMetricPayloadBundler, + packager_factory=_StaticMetricBundlePackager, ) register_metric_bundle_kind( "test-strict", payload_type=_StrictMetricPayload, - payload_bundler_factory=_StaticMetricPayloadBundler, + packager_factory=_StaticMetricBundlePackager, ) @@ -292,12 +293,12 @@ def test_parse_required_workspace_name_rejects_extra_separator() -> None: def test_evaluate_job_hydrates_mixed_bundle_kinds_by_payload_kind() -> None: - """Execution-side hydration dispatches per bundle instead of assuming one bundler.""" + """Execution-side hydration dispatches per bundle instead of assuming one packager.""" cloudpickle_bundle = bundle_metric( ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - CloudpickleMetricPayloadBundler(), + CloudpickleMetricBundlePackager(), ) - static_bundle = bundle_metric(_StaticMetric("test-static"), _StaticMetricPayloadBundler()) + static_bundle = bundle_metric(_StaticMetric("test-static"), _StaticMetricBundlePackager()) metrics = EvaluateJob._hydrate_metrics([cloudpickle_bundle, static_bundle]) @@ -495,6 +496,34 @@ async def test_evaluate_job_compile_injects_metric_and_target_secrets() -> None: assert secrets == {"NVIDIA_BUILD_API_KEY": "NVIDIA_BUILD_API_KEY"} +async def test_evaluate_job_compile_rejects_secret_reserved_env_names() -> None: + metric = LLMJudgeMetric( + model=Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/nemotron-3-super-120b-a12b", + api_key_secret=SecretRef(root=PERSISTENT_JOB_STORAGE_PATH_ENVVAR), + ), + scores=[ + RangeScore( + name="quality", + minimum=1, + maximum=5, + parser=JSONScoreParser(json_path="quality"), + ), + ], + ) + spec = EvaluateSpec.model_validate({**_exact_match_spec(), "metrics": [_bundle_payload(metric)]}) + + with pytest.raises(ValueError, match="reserved"): + await EvaluateJob.compile( + workspace="default", + spec=spec, + entity_client=object(), + job_name=None, + async_sdk=object(), + ) + + class TestEvaluateSpec: """Validation coverage for evaluator job specs.""" @@ -744,7 +773,7 @@ async def test_rejects_wrong_offline_param_type(self, mocker: MockerFixture) -> async_sdk=object(), ) - async def test_fileset_ref_dataset_compiles_into_bundle_native_step(self) -> None: + async def test_fileset_ref_dataset_compiles_into_evaluate_step(self) -> None: dataset = FilesetRef(root="default/helpsteer2#validation/*.jsonl") compiled = await EvaluateJob.compile( @@ -756,12 +785,8 @@ async def test_fileset_ref_dataset_compiles_into_bundle_native_step(self) -> Non ) job_spec = PlatformJobSpec.model_validate(compiled) - assert [step.name for step in job_spec.steps] == ["dataset-download", "evaluate"] - download_step = job_spec.steps[0] - download_container = cast(Any, download_step.executor).container - assert download_container.entrypoint == ["python", "-m", "nmp.evaluator.tasks.download_fileset"] - assert download_container.command[-2:] == ["--dataset", dataset.model_dump_json()] - config = cast(dict[str, Any], job_spec.steps[1].config) + assert [step.name for step in job_spec.steps] == ["evaluate"] + config = cast(dict[str, Any], job_spec.steps[0].config) assert config["dataset"] == dataset.root diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py index aef0eb2dfe..93b6e5dbb5 100644 --- a/plugins/nemo-evaluator/tests/test_sdk.py +++ b/plugins/nemo-evaluator/tests/test_sdk.py @@ -14,7 +14,7 @@ from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec from nemo_evaluator.sdk import http_utils from nemo_evaluator.sdk._executor import ( - MetricPayloadBundlerPolicyError, + MetricBundlePackagerPolicyError, _AsyncEvaluatorPluginExecutor, _build_evaluate_spec, _SyncEvaluatorPluginExecutor, @@ -25,12 +25,12 @@ from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, + MetricBundlePackager, MetricBundlePayload, MetricBundlingError, - MetricPayloadBundler, bundle_metric, ) -from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.protocol import Metric @@ -47,7 +47,7 @@ "metrics": [ bundle_metric( _EXACT_MATCH_METRIC, - CloudpickleMetricPayloadBundler(), + CloudpickleMetricBundlePackager(), ).model_dump(mode="json") ], "dataset": [{"expected": "a", "output": "a"}], @@ -71,20 +71,20 @@ def _single_metric(spec: EvaluateSpec) -> MetricBundle: return spec.metrics[0] -class _RecordingMetricPayloadBundler(MetricPayloadBundler): - """Test bundler that records all runtime metrics selected for bundling.""" +class _RecordingMetricBundlePackager(MetricBundlePackager): + """Test packager that records all runtime metrics selected for packaging.""" def __init__(self) -> None: self.metrics: list[Metric] = [] - self._delegate = CloudpickleMetricPayloadBundler() + self._delegate = CloudpickleMetricBundlePackager() - def bundle(self, metric: Metric) -> MetricBundlePayload: + def package(self, metric: Metric) -> MetricBundlePayload: self.metrics.append(metric) - return self._delegate.bundle(metric) + return self._delegate.package(metric) - def unbundle(self, payload: MetricBundlePayload) -> Metric: + def load(self, payload: MetricBundlePayload) -> Metric: del payload - raise NotImplementedError("test bundler only exercises submission-side bundling") + raise NotImplementedError("test packager only exercises submission-side packaging") class _SyncPlatform: @@ -187,11 +187,11 @@ def test_resolve_workspace_requires_explicit_or_default_workspace() -> None: def test_bundle_metrics_for_spec_rejects_non_metric_object() -> None: """Metrics must satisfy the runtime Metric protocol before plugin execution.""" with pytest.raises(MetricBundlingError, match="Metric protocol"): - bundle_metrics_for_spec(cast(Any, object()), metric_payload_bundler=CloudpickleMetricPayloadBundler()) + bundle_metrics_for_spec(cast(Any, object()), metric_bundle_packager=CloudpickleMetricBundlePackager()) -def test_build_evaluate_spec_requires_metric_payload_bundler() -> None: - with pytest.raises(MetricPayloadBundlerPolicyError, match="CloudpickleMetricPayloadBundler"): +def test_build_evaluate_spec_requires_metric_bundle_packager() -> None: + with pytest.raises(MetricBundlePackagerPolicyError, match="CloudpickleMetricBundlePackager"): _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), request=EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]), @@ -203,7 +203,7 @@ def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: model = Model(url="https://model.test/v1", name="model-a") spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), request=EvaluationRequest( dataset=[{"expected": "a", "output": "a"}], target=model, @@ -215,19 +215,19 @@ def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: assert spec.prompt_template == "Answer: {{item.input}}" -def test_build_evaluate_spec_uses_selected_bundler_for_all_runtime_metrics() -> None: - """Submission bundles all outgoing runtime metrics with the caller-selected bundler.""" +def test_build_evaluate_spec_uses_selected_packager_for_all_runtime_metrics() -> None: + """Submission packages all outgoing runtime metrics with the caller-selected packager.""" metric_a = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") metric_b = ExactMatchMetric(reference="{{item.other_expected}}", candidate="{{item.other_output}}") - bundler = _RecordingMetricPayloadBundler() + packager = _RecordingMetricBundlePackager() spec = _build_evaluate_spec( metrics=[metric_a, metric_b], - metric_payload_bundler=bundler, + metric_bundle_packager=packager, request=EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]), ) - assert bundler.metrics == [metric_a, metric_b] + assert packager.metrics == [metric_a, metric_b] assert [metric.metric_type for metric in spec.metrics] == ["exact-match", "exact-match"] @@ -235,7 +235,7 @@ def test_build_evaluate_spec_excludes_aggregate_fields() -> None: """Evaluator specs should not persist result-shaping options.""" spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), request=EvaluationRequest( dataset=[{"expected": "a", "output": "a"}], params=RunConfig(), @@ -253,7 +253,7 @@ def test_build_evaluate_spec_preserves_fileset_ref_dataset() -> None: spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), request=EvaluationRequest(dataset=cast(Any, dataset)), ) @@ -264,7 +264,7 @@ def test_build_evaluate_spec_synthesizes_fileset_ref_fragment_from_dataset_glob_ """FilesetRef datasets should encode dataset_glob_pattern as the existing fragment selector syntax.""" spec = _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), request=EvaluationRequest( dataset=cast(Any, FilesetRef(root="default/helpsteer2")), dataset_glob_pattern="validation/*.jsonl", @@ -279,7 +279,7 @@ def test_build_evaluate_spec_rejects_fileset_ref_fragment_and_dataset_glob_patte with pytest.raises(ValueError, match=r"dataset_glob_pattern.*FilesetRef"): _build_evaluate_spec( metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), request=EvaluationRequest( dataset=cast(Any, FilesetRef(root="default/helpsteer2#validation/*.jsonl")), dataset_glob_pattern="train/*.jsonl", @@ -509,6 +509,8 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non model = Model(url="https://model.test/v1", name="model-a") config = RunConfigOnlineModel(parallelism=3, limit_samples=5) + packager = CloudpickleMetricBundlePackager() + job = resource.submit( metric=metric, dataset=dataset, @@ -516,7 +518,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) assert job is expected_job @@ -527,7 +529,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: @@ -539,7 +541,9 @@ def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") dataset = FilesetRef(root="default/helpsteer2") - job = resource.submit(metric=metric, dataset=dataset) + packager = CloudpickleMetricBundlePackager() + + job = resource.submit(metric=metric, dataset=dataset, metric_bundle_packager=packager) assert job is expected_job submit.assert_called_once_with( @@ -549,9 +553,19 @@ def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: target=None, dataset_glob_pattern=None, prompt_template=None, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) + def test_requires_metric_bundle_packager(self) -> None: + """Submit should fail fast before delegating without a remote metric packager.""" + resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) + + with pytest.raises(ValueError, match="metric_bundle_packager is required"): + resource.submit( + metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + dataset=[{"expected": "a", "output": "a"}], + ) + class TestEvaluatorRun: """Tests for ``Evaluator.run`` executor delegation.""" @@ -630,7 +644,7 @@ def test_run_uses_local_executor_execution(self, mocker: MockerFixture) -> None: remote_evaluate.assert_not_called() -def test_sync_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None: +def test_sync_executor_evaluate_calls_sdk_directly_without_packaging(mocker: MockerFixture) -> None: platform = _SyncPlatform() executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) @@ -708,7 +722,7 @@ def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: Mocke result = executor.evaluate_remote( metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), request=request, - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) assert result == expected @@ -909,6 +923,8 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) model = Model(url="https://model.test/v1", name="model-a") config = RunConfigOnlineModel(parallelism=3, limit_samples=5) + packager = CloudpickleMetricBundlePackager() + job = await resource.submit( metric=metric, dataset=dataset, @@ -916,7 +932,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) assert job is expected_job @@ -927,7 +943,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) target=model, dataset_glob_pattern="*.jsonl", prompt_template={"template": "Answer {{item.input}}"}, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) @pytest.mark.asyncio @@ -940,7 +956,9 @@ async def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") dataset = FilesetRef(root="default/helpsteer2") - job = await resource.submit(metric=metric, dataset=dataset) + packager = CloudpickleMetricBundlePackager() + + job = await resource.submit(metric=metric, dataset=dataset, metric_bundle_packager=packager) assert job is expected_job submit.assert_awaited_once_with( @@ -950,9 +968,20 @@ async def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: target=None, dataset_glob_pattern=None, prompt_template=None, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) + @pytest.mark.asyncio + async def test_requires_metric_bundle_packager(self) -> None: + """Submit should fail fast before delegating without a remote metric packager.""" + resource = AsyncEvaluator(cast(AsyncNeMoPlatform, _AsyncPlatform())) + + with pytest.raises(ValueError, match="metric_bundle_packager is required"): + await resource.submit( + metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), + dataset=[{"expected": "a", "output": "a"}], + ) + class TestAsyncEvaluatorRun: """Tests for ``AsyncEvaluator.run`` executor delegation.""" @@ -1060,7 +1089,7 @@ async def test_async_executor_remote_submit_uses_platform_async_client_headers_a @pytest.mark.asyncio -async def test_async_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None: +async def test_async_executor_evaluate_calls_sdk_directly_without_packaging(mocker: MockerFixture) -> None: platform = _AsyncPlatform() executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) @@ -1105,7 +1134,7 @@ async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker result = await executor.evaluate_remote( metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), request=request, - metric_payload_bundler=CloudpickleMetricPayloadBundler(), + metric_bundle_packager=CloudpickleMetricBundlePackager(), ) assert result == expected diff --git a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py index 1458aff7b1..845d52e8a6 100644 --- a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py +++ b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py @@ -30,7 +30,7 @@ metric_job_status_value, ) from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric -from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayloadBundler +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, @@ -50,7 +50,7 @@ "metrics": [ bundle_metric( ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - CloudpickleMetricPayloadBundler(), + CloudpickleMetricBundlePackager(), ).model_dump(mode="json") ], "dataset": [{"expected": "a", "output": "a"}], diff --git a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py index 85a2ff5d59..2b9c458295 100644 --- a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py +++ b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py @@ -13,6 +13,7 @@ import pytest from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend, NMPBackend +from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator_sdk.execution.backends.base import EvaluationBackend, SyncEvaluationBackend from nemo_evaluator_sdk.execution.config import EvaluationRequest from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric @@ -103,17 +104,33 @@ def test_evaluate_remote_delegates_to_resource_executor_remote_path(self, mocker remote_evaluate = mocker.patch.object(resource._executor, "evaluate_remote", return_value=expected) metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") request = EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]) + packager = CloudpickleMetricBundlePackager() - result = NMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) + result = NMPBackend(resource, execution_mode="remote", metric_bundle_packager=packager).evaluate( + metric=metric, request=request + ) assert result is expected remote_evaluate.assert_called_once_with( metric=metric, request=request, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) local_evaluate.assert_not_called() + def test_evaluate_remote_requires_metric_bundle_packager(self, mocker: MockerFixture) -> None: + resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) + local_evaluate = mocker.patch.object(resource._executor, "evaluate") + remote_evaluate = mocker.patch.object(resource._executor, "evaluate_remote") + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + request = EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]) + + with pytest.raises(ValueError, match="metric_bundle_packager is required"): + NMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) + + remote_evaluate.assert_not_called() + local_evaluate.assert_not_called() + def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: MockerFixture) -> None: resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) expected = _empty_benchmark_result() @@ -209,17 +226,34 @@ async def test_evaluate_remote_delegates_to_resource_executor_remote_path(self, ) metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") request = EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]) + packager = CloudpickleMetricBundlePackager() - result = await AsyncNMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) + result = await AsyncNMPBackend(resource, execution_mode="remote", metric_bundle_packager=packager).evaluate( + metric=metric, request=request + ) assert result is expected remote_evaluate.assert_awaited_once_with( metric=metric, request=request, - metric_payload_bundler=None, + metric_bundle_packager=packager, ) local_evaluate.assert_not_awaited() + @pytest.mark.asyncio + async def test_evaluate_remote_requires_metric_bundle_packager(self, mocker: MockerFixture) -> None: + resource = AsyncEvaluator(cast(AsyncNeMoPlatform, _AsyncPlatform())) + local_evaluate = mocker.patch.object(resource._executor, "evaluate", new=AsyncMock()) + remote_evaluate = mocker.patch.object(resource._executor, "evaluate_remote", new=AsyncMock()) + metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") + request = EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]) + + with pytest.raises(ValueError, match="metric_bundle_packager is required"): + await AsyncNMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request) + + remote_evaluate.assert_not_awaited() + local_evaluate.assert_not_awaited() + @pytest.mark.asyncio async def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: MockerFixture) -> None: resource = AsyncEvaluator(cast(AsyncNeMoPlatform, _AsyncPlatform()))