diff --git a/packages/nemo_evaluator_sdk/pyproject.toml b/packages/nemo_evaluator_sdk/pyproject.toml
index 35988f2dbb..ff2d47f6d2 100644
--- a/packages/nemo_evaluator_sdk/pyproject.toml
+++ b/packages/nemo_evaluator_sdk/pyproject.toml
@@ -16,6 +16,7 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
+  "cloudpickle>=3.1.1",
   "pydantic>=2.10.6",
   "jinja2>=3.1.6",
   "jsonschema>=4.23.0",
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py
index 9b9d07e44b..6c58887e4f 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/__init__.py
@@ -12,13 +12,22 @@
     EvaluationError,
     EvaluationPhase,
 )
+from nemo_evaluator_sdk.metrics.base import (
+    MetricBundle,
+    MetricBundlePayload,
+    MetricBundler,
+    MetricBundlingError,
+    MetricMetadata,
+)
 from nemo_evaluator_sdk.metrics.bleu import BLEUMetric
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler, CloudpickleMetricPayload
 from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
 from nemo_evaluator_sdk.metrics.f1 import F1Metric
 from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
 from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric
 from nemo_evaluator_sdk.metrics.protocol import (
     Metric,
+    MetricTypeName,
     validate_metric_result,
 )
 from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric
@@ -69,6 +78,10 @@
 __all__ = [
     "BLEUMetric",
     "Agent",
+    "MetricBundle",
+    "MetricBundlePayload",
+    "CloudpickleMetricBundler",
+    "CloudpickleMetricPayload",
     "EvaluationError",
     "EvaluationPhase",
     "DatasetLoadError",
@@ -85,11 +98,15 @@
     "InferenceStructuredOutput",
     "JSONScoreParser",
     "Metric",
+    "MetricBundler",
+    "MetricBundlingError",
     "MetricDescriptor",
     "MetricInput",
+    "MetricMetadata",
     "MetricOutput",
     "MetricOutputSpec",
     "MetricResult",
+    "MetricTypeName",
     "LLMJudgeMetric",
     "BooleanValue",
     "CandidateOutput",
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py
new file mode 100644
index 0000000000..5c0b073062
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/base.py
@@ -0,0 +1,185 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Backend-neutral metric bundle models and protocols."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Mapping
+from typing import Any, Literal, Protocol, cast
+
+from nemo_evaluator_sdk.metrics.protocol import Metric, MetricOutputSpec, MetricTypeName, MetricWithSecrets
+from nemo_evaluator_sdk.values.common import SecretRef
+from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, field_serializer, field_validator, model_validator
+
+
+class MetricBundlingError(ValueError):
+    """Raised when a metric cannot be bundled or hydrated."""
+
+
+class MetricMetadata(BaseModel):
+    """User-facing metadata captured with a bundled metric."""
+
+    model_config = ConfigDict(extra="allow", revalidate_instances="never")
+
+    description: str | None = None
+    labels: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("labels")
+    @classmethod
+    def _labels_must_be_strings(cls, value: dict[str, str]) -> dict[str, str]:
+        for key, label_value in value.items():
+            if not isinstance(key, str) or not isinstance(label_value, str):
+                raise ValueError("metric labels must be a mapping of string keys to string values")
+        return value
+
+
+class BundledMetricOutputSpec(BaseModel):
+    """JSON-safe projection of a runtime metric output spec."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    description: str | None = None
+    value_json_schema: dict[str, Any]
+
+    @classmethod
+    def from_output_spec(cls, output: MetricOutputSpec) -> "BundledMetricOutputSpec":
+        """Capture the serializable contract for one runtime output."""
+        return cls(
+            name=output.name,
+            description=output.description,
+            value_json_schema=output.value_json_schema(),
+        )
+
+
+class MetricBundlePayload(BaseModel, ABC):
+    """Base class for concrete Pydantic metric bundle payloads."""
+
+    @property
+    @abstractmethod
+    def kind(self) -> str:
+        """Payload discriminator used to select the bundler implementation."""
+        ...
+
+
+_PAYLOAD_TYPES: dict[str, type[MetricBundlePayload]] = {}
+_BUNDLER_FACTORIES: dict[str, Callable[[], MetricBundler]] = {}
+
+
+def _payload_kind(payload: MetricBundlePayload) -> str:
+    kind = payload.kind
+    if not kind:
+        raise MetricBundlingError("metric bundle payload kind must not be empty")
+    return kind
+
+
+def register_metric_bundle_payload(kind: str, payload_type: type[MetricBundlePayload]) -> None:
+    """Register a concrete Pydantic payload model for a bundle kind."""
+    if not kind:
+        raise ValueError("metric bundle payload kind must not be empty")
+    _PAYLOAD_TYPES[kind] = payload_type
+
+
+def register_metric_bundler(kind: str, factory: Callable[[], MetricBundler]) -> None:
+    """Register a metric bundler factory for a payload kind."""
+    if not kind:
+        raise ValueError("metric bundle payload kind must not be empty")
+    _BUNDLER_FACTORIES[kind] = factory
+
+
+class MetricBundle(BaseModel):
+    """Standalone executable metric bundle entity used by backend execution."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    bundle_kind: Literal["metric-bundle"] = "metric-bundle"
+    bundle_format_version: Literal["v1"] = "v1"
+    metric_type: MetricTypeName
+    metadata: MetricMetadata = Field(default_factory=MetricMetadata)
+    outputs: list[BundledMetricOutputSpec] = Field(min_length=1)
+    secrets: dict[str, SecretRef] = Field(default_factory=dict)
+    payload: SerializeAsAny[MetricBundlePayload]
+    digest: str
+
+    @field_serializer("payload")
+    def _serialize_payload(self, payload: MetricBundlePayload) -> dict[str, Any]:
+        value = payload.model_dump(mode="json")
+        value["kind"] = _payload_kind(payload)
+        return value
+
+    @field_validator("payload", mode="before")
+    @classmethod
+    def _payload_must_have_kind(cls, value: object) -> object:
+        if isinstance(value, MetricBundlePayload):
+            return value
+        if not isinstance(value, Mapping):
+            raise ValueError("metric bundle payload must be an object")
+        payload_data = cast(Mapping[str, object], value)
+        kind = payload_data.get("kind")
+        if not isinstance(kind, str) or not kind:
+            raise ValueError("metric bundle payload must include a non-empty kind")
+        payload_type = _PAYLOAD_TYPES.get(kind)
+        if payload_type is None:
+            raise ValueError(f"unsupported metric bundle payload kind: {kind}")
+        return payload_type.model_validate(value)
+
+    @model_validator(mode="after")
+    def _output_names_must_be_unique(self) -> "MetricBundle":
+        names = [output.name for output in self.outputs]
+        duplicates = sorted({name for name in names if names.count(name) > 1})
+        if duplicates:
+            raise ValueError(f"duplicate metric output names: {duplicates}")
+        return self
+
+
+class MetricBundler(Protocol):
+    """Interface for metric bundle implementations."""
+
+    def bundle(self, metric: Metric) -> MetricBundle:
+        """Serialize an executable metric to a bundle entity."""
+        ...
+
+    def unbundle(self, metric: MetricBundle) -> Metric:
+        """Hydrate an executable metric from a bundle entity."""
+        ...
+
+
+def metric_bundler_for_payload(payload: MetricBundlePayload) -> MetricBundler:
+    """Create the bundler registered for a metric bundle payload."""
+    kind = _payload_kind(payload)
+    factory = _BUNDLER_FACTORIES.get(kind)
+    if factory is None:
+        raise MetricBundlingError(f"unsupported metric bundle payload kind: {kind}")
+    return factory()
+
+
+def validate_metric_type(metric: Metric) -> str:
+    """Return the runtime metric type after validating the protocol contract."""
+    value = metric.type
+    if not isinstance(value, str):
+        raise MetricBundlingError("metric type must be a string")
+    if not value:
+        raise MetricBundlingError("metric type must not be empty")
+    return value
+
+
+def metric_metadata(metric: Metric) -> MetricMetadata:
+    """Capture optional runtime metric metadata."""
+    description = getattr(metric, "description", None)
+    if description is not None and not isinstance(description, str):
+        raise MetricBundlingError("metric description must be a string when provided")
+
+    raw_labels = getattr(metric, "labels", None) or {}
+    if not isinstance(raw_labels, Mapping):
+        raise MetricBundlingError("metric labels must be a mapping when provided")
+    labels = dict(raw_labels)
+    return MetricMetadata(description=description, labels=labels)
+
+
+def metric_secrets(metric: Metric) -> dict[str, SecretRef]:
+    """Capture secret environment mappings needed to execute one metric."""
+    if not isinstance(metric, MetricWithSecrets):
+        return {}
+    return metric.secrets()
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/cloudpickle.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/cloudpickle.py
new file mode 100644
index 0000000000..b6156a91d9
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/cloudpickle.py
@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Cloudpickle-backed metric bundle implementation."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import pickle
+import platform
+from typing import Literal
+
+import cloudpickle
+from nemo_evaluator_sdk.metrics.base import (
+    BundledMetricOutputSpec,
+    MetricBundle,
+    MetricBundlePayload,
+    MetricBundler,
+    MetricBundlingError,
+    metric_metadata,
+    metric_secrets,
+    register_metric_bundle_payload,
+    register_metric_bundler,
+    validate_metric_type,
+)
+from nemo_evaluator_sdk.metrics.protocol import Metric
+from pydantic import ConfigDict, field_validator
+
+
+class CloudpickleMetricPayload(MetricBundlePayload):
+    """Cloudpickle payload for an executable metric object."""
+
+    model_config = ConfigDict(extra="ignore", ser_json_bytes="base64", val_json_bytes="base64")
+
+    python_version: str
+    cloudpickle_version: str
+    pickle_protocol: int
+    blob: bytes
+
+    @property
+    def kind(self) -> Literal["cloudpickle"]:
+        """Payload discriminator used by the metric bundle registry."""
+        return "cloudpickle"
+
+    @classmethod
+    def from_blob(cls, blob: bytes) -> CloudpickleMetricPayload:
+        """Create a JSON-safe cloudpickle payload from raw bytes."""
+        return cls(
+            python_version=platform.python_version(),
+            cloudpickle_version=cloudpickle.__version__,
+            pickle_protocol=pickle.HIGHEST_PROTOCOL,
+            blob=blob,
+        )
+
+    @field_validator("blob")
+    @classmethod
+    def _blob_must_not_be_empty(cls, value: bytes) -> bytes:
+        if not value:
+            raise ValueError("blob must not be empty")
+        try:
+            encoded = base64.b64encode(value).decode("ascii")
+            base64.b64decode(encoded.encode("ascii"), validate=True)
+        except Exception as e:
+            raise ValueError("blob must be JSON-serializable as base64") from e
+        return value
+
+    def blob_bytes(self) -> bytes:
+        """Return the decoded cloudpickle payload bytes."""
+        return self.blob
+
+
+class CloudpickleMetricBundler(MetricBundler):
+    """Cloudpickle-backed metric bundler.
+
+    Cloudpickle bundles execute arbitrary Python code when hydrated. This
+    implementation is intended for explicit opt-in development/MVP use.
+    """
+
+    def bundle(self, metric: Metric) -> MetricBundle:
+        """Serialize a runtime metric object to a cloudpickle bundle."""
+        if not isinstance(metric, Metric):
+            raise MetricBundlingError("object does not satisfy the Metric protocol")
+
+        blob = cloudpickle.dumps(metric, protocol=pickle.HIGHEST_PROTOCOL)
+        digest = hashlib.sha256(blob).hexdigest()
+        return MetricBundle(
+            metric_type=validate_metric_type(metric),
+            metadata=metric_metadata(metric),
+            outputs=[BundledMetricOutputSpec.from_output_spec(output) for output in metric.output_spec()],
+            secrets=metric_secrets(metric),
+            payload=CloudpickleMetricPayload.from_blob(blob),
+            digest=digest,
+        )
+
+    def unbundle(self, metric: MetricBundle) -> Metric:
+        """Hydrate a metric from a cloudpickle bundle."""
+        payload = CloudpickleMetricPayload.model_validate(metric.payload.model_dump(mode="python"))
+        blob = payload.blob_bytes()
+        digest = hashlib.sha256(blob).hexdigest()
+        if digest != metric.digest:
+            raise MetricBundlingError("metric bundle digest does not match payload")
+
+        hydrated_metric = cloudpickle.loads(blob)
+        if not isinstance(hydrated_metric, Metric):
+            raise MetricBundlingError("unbundled object does not satisfy the Metric protocol")
+
+        output_names = [output.name for output in hydrated_metric.output_spec()]
+        bundled_output_names = [output.name for output in metric.outputs]
+        if output_names != bundled_output_names:
+            raise MetricBundlingError("unbundled metric output spec does not match bundle metadata")
+        if validate_metric_type(hydrated_metric) != metric.metric_type:
+            raise MetricBundlingError("unbundled metric type does not match bundle metadata")
+        return hydrated_metric
+
+
+register_metric_bundle_payload("cloudpickle", CloudpickleMetricPayload)
+register_metric_bundler("cloudpickle", CloudpickleMetricBundler)
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py
index f93f9a34bd..a0ed133d7b 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/protocol.py
@@ -7,12 +7,13 @@
 
 import math
 from collections.abc import Awaitable, Callable
-from typing import Any, Protocol, runtime_checkable
+from typing import Annotated, Any, Protocol, runtime_checkable
 
 from nemo_evaluator_sdk.values.common import SecretRef
-from pydantic import BaseModel, ConfigDict, Field, RootModel, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, RootModel, StringConstraints, field_serializer, field_validator
 
 SecretResolver = Callable[[str], Awaitable[str | None]]
+MetricTypeName = Annotated[str, StringConstraints(min_length=1)]
 
 
 class DatasetRow(BaseModel):
@@ -126,16 +127,9 @@ class MetricDescriptor(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    type: str
+    type: MetricTypeName
     outputs: list[MetricOutputSpec] = Field(min_length=1)
 
-    @field_validator("type")
-    @classmethod
-    def _type_must_not_be_empty(cls, value: str) -> str:
-        if not value:
-            raise ValueError("metric type must not be empty")
-        return value
-
     @field_validator("outputs")
     @classmethod
     def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]:
@@ -174,7 +168,7 @@ class Metric(Protocol):
     """Shared row-scoring primitive for SDK runtime metrics."""
 
     @property
-    def type(self) -> str:
+    def type(self) -> MetricTypeName:
         """Return the public metric key/type identifier."""
         ...
 
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py
index f6d6c68699..cbb9df6bb4 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/__init__.py
@@ -15,6 +15,7 @@
     MetricOutput,
     MetricOutputSpec,
     MetricResult,
+    MetricTypeName,
 )
 from nemo_evaluator_sdk.values.agents import Agent
 from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes
@@ -114,6 +115,7 @@
     "MetricOutput",
     "MetricOutputSpec",
     "MetricResult",
+    "MetricTypeName",
     "MetricScore",
     "Model",
     "DatasetInput",
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/metrics.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/metrics.py
index 7f8aa995db..fd094e0b83 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/metrics.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/metrics.py
@@ -20,6 +20,7 @@
 from nemo_evaluator_sdk.dataset_schemas.compatibility import merge_metric_required_schemas
 from nemo_evaluator_sdk.dataset_schemas.templates import infer_required_schema_from_template
 from nemo_evaluator_sdk.enums import MetricType
+from nemo_evaluator_sdk.metrics.protocol import MetricTypeName
 from nemo_evaluator_sdk.values.common import SecretRef, SupportedJobTypes
 from nemo_evaluator_sdk.values.dataset_schemas import InputSchema
 from nemo_evaluator_sdk.values.models import Model, ReasoningParams
@@ -129,7 +130,7 @@ class MetricBase(BaseModel):
 
     __entity_type__: ClassVar[str] = "metric"
 
-    type: str = Field(description="The type of metric. Used as a discriminator for the metric type.")
+    type: MetricTypeName = Field(description="The type of metric. Used as a discriminator for the metric type.")
     description: str | None = Field(default=None, description="Human-readable description of the metric.")
     labels: dict[str, str] = Field(
         default_factory=dict, description="Labels are key-value pairs that can be used for grouping and filtering."
diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_cloudpickle.py b/packages/nemo_evaluator_sdk/tests/metrics/test_cloudpickle.py
new file mode 100644
index 0000000000..7df5561233
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/tests/metrics/test_cloudpickle.py
@@ -0,0 +1,247 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+
+import pytest
+from nemo_evaluator_sdk.enums import ModelFormat
+from nemo_evaluator_sdk.metrics.base import (
+    MetricBundle,
+    MetricBundlingError,
+)
+from nemo_evaluator_sdk.metrics.bleu import BLEUMetric
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler, CloudpickleMetricPayload
+from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+from nemo_evaluator_sdk.metrics.f1 import F1Metric
+from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
+from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric
+from nemo_evaluator_sdk.metrics.protocol import (
+    Metric,
+    MetricInput,
+    MetricOutput,
+    MetricOutputSpec,
+    MetricResult,
+)
+from nemo_evaluator_sdk.metrics.ragas import (
+    AgentGoalAccuracyMetric,
+    AnswerAccuracyMetric,
+    ContextEntityRecallMetric,
+    ContextPrecisionMetric,
+    ContextRecallMetric,
+    ContextRelevanceMetric,
+    FaithfulnessMetric,
+    NoiseSensitivityMetric,
+    ResponseGroundednessMetric,
+    ResponseRelevancyMetric,
+    ToolCallAccuracyMetric,
+    TopicAdherenceMetric,
+)
+from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric
+from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric
+from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric
+from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric
+from nemo_evaluator_sdk.values import Model, SecretRef
+from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore, RemoteScore
+
+
+class _CustomMetric:
+    type = "custom-score"
+    description = "custom metric"
+    labels = {"source": "test"}
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [MetricOutputSpec.continuous_score("score")]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        del input
+        return MetricResult(outputs=[MetricOutput(name="score", value=1.0)])
+
+
+class _NotMetric:
+    pass
+
+
+class _EmptyTypeMetric(_CustomMetric):
+    type = ""
+
+
+def _judge_model() -> Model:
+    return Model(
+        url="https://judge.example.test/v1/chat/completions",
+        name="judge-model",
+        format=ModelFormat.OPEN_AI,
+    )
+
+
+def _embeddings_model() -> Model:
+    return Model(
+        url="https://judge.example.test/v1/embeddings",
+        name="embedding-model",
+        format=ModelFormat.OPEN_AI,
+    )
+
+
+def _builtin_metric_cases() -> Sequence[tuple[str, Metric]]:
+    judge_model = _judge_model()
+    return [
+        ("exact_match", ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")),
+        ("f1", F1Metric(reference="{{item.expected}}", candidate="{{item.output}}")),
+        ("bleu", BLEUMetric(references=["{{item.expected}}"], candidate="{{item.output}}")),
+        ("rouge", ROUGEMetric(reference="{{item.expected}}", candidate="{{item.output}}")),
+        (
+            "string_check",
+            StringCheckMetric(
+                operation="contains", left_template="{{item.output}}", right_template="{{item.expected}}"
+            ),
+        ),
+        (
+            "number_check",
+            NumberCheckMetric(operation="equals", left_template="{{item.left}}", right_template="{{item.right}}"),
+        ),
+        ("tool_calling", ToolCallingMetric(reference="{{item.expected_tool_calls}}")),
+        (
+            "llm_judge",
+            LLMJudgeMetric(
+                model=judge_model,
+                scores=[
+                    RangeScore(
+                        name="helpfulness",
+                        minimum=1,
+                        maximum=5,
+                        parser=JSONScoreParser(json_path="helpfulness"),
+                    )
+                ],
+                prompt_template="Judge: {{item.expected}} -> {{item.output}}",
+            ),
+        ),
+        (
+            "remote",
+            RemoteMetric(
+                url="https://remote.example.test",
+                body={"prompt": "{{item.prompt}}"},
+                scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))],
+            ),
+        ),
+        (
+            "nemo_agent_toolkit_remote",
+            NemoAgentToolkitRemoteMetric(url="https://remote.example.test", evaluator_name="nat-quality"),
+        ),
+        ("topic_adherence", TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model)),
+        ("tool_call_accuracy", ToolCallAccuracyMetric()),
+        ("agent_goal_accuracy", AgentGoalAccuracyMetric(judge_model=judge_model)),
+        ("answer_accuracy", AnswerAccuracyMetric(judge_model=judge_model)),
+        ("context_relevance", ContextRelevanceMetric(judge_model=judge_model)),
+        ("response_groundedness", ResponseGroundednessMetric(judge_model=judge_model)),
+        ("context_recall", ContextRecallMetric(judge_model=judge_model)),
+        ("context_precision", ContextPrecisionMetric(judge_model=judge_model)),
+        ("context_entity_recall", ContextEntityRecallMetric(judge_model=judge_model)),
+        (
+            "response_relevancy",
+            ResponseRelevancyMetric(judge_model=judge_model, embeddings_model=_embeddings_model()),
+        ),
+        ("faithfulness", FaithfulnessMetric(judge_model=judge_model)),
+        ("noise_sensitivity", NoiseSensitivityMetric(judge_model=judge_model)),
+    ]
+
+
+def test_cloudpickle_bundler_round_trips_builtin_metric() -> None:
+    metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
+    bundler = CloudpickleMetricBundler()
+
+    bundle = bundler.bundle(metric)
+    hydrated = bundler.unbundle(bundle)
+
+    assert bundle.metric_type == "exact-match"
+    assert bundle.outputs[0].name == "exact-match"
+    assert isinstance(hydrated, ExactMatchMetric)
+
+
+@pytest.mark.parametrize(
+    ("case_name", "metric"), _builtin_metric_cases(), ids=[case[0] for case in _builtin_metric_cases()]
+)
+def test_cloudpickle_bundler_round_trips_every_builtin_metric(case_name: str, metric: Metric) -> None:
+    bundler = CloudpickleMetricBundler()
+
+    bundle = bundler.bundle(metric)
+    restored = MetricBundle.model_validate_json(bundle.model_dump_json())
+    hydrated = bundler.unbundle(restored)
+
+    assert restored.metric_type
+    assert restored.outputs
+    assert [output.name for output in hydrated.output_spec()] == [output.name for output in metric.output_spec()]
+    assert type(hydrated) is type(metric), case_name
+
+
+def test_cloudpickle_bundler_round_trips_custom_protocol_metric() -> None:
+    bundler = CloudpickleMetricBundler()
+
+    bundle = bundler.bundle(_CustomMetric())
+    serialized = bundle.model_dump_json()
+    restored = MetricBundle.model_validate_json(serialized)
+    hydrated = bundler.unbundle(restored)
+
+    assert restored.bundle_kind == "metric-bundle"
+    assert restored.metric_type == "custom-score"
+    assert restored.metadata.description == "custom metric"
+    assert restored.metadata.labels == {"source": "test"}
+    assert restored.outputs[0].name == "score"
+    assert isinstance(hydrated, _CustomMetric)
+
+
+def test_cloudpickle_bundler_captures_metric_secrets() -> None:
+    metric = LLMJudgeMetric(
+        model=Model(
+            url="https://judge.example.test/v1/chat/completions",
+            name="judge-model",
+            api_key_secret=SecretRef(root="judge-secret"),
+            format=ModelFormat.OPEN_AI,
+        ),
+        scores=[
+            RangeScore(
+                name="helpfulness",
+                minimum=1,
+                maximum=5,
+                parser=JSONScoreParser(json_path="helpfulness"),
+            )
+        ],
+    )
+
+    bundle = CloudpickleMetricBundler().bundle(metric)
+    restored = MetricBundle.model_validate_json(bundle.model_dump_json())
+
+    assert restored.secrets == {"judge_secret": SecretRef(root="judge-secret")}
+
+
+def test_cloudpickle_bundler_captures_digest_and_payload_metadata() -> None:
+    bundle = CloudpickleMetricBundler().bundle(_CustomMetric())
+    payload = CloudpickleMetricPayload.model_validate(bundle.payload)
+    blob = payload.blob_bytes()
+
+    assert bundle.digest == hashlib.sha256(blob).hexdigest()
+    assert payload.kind == "cloudpickle"
+    assert payload.python_version
+    assert payload.cloudpickle_version
+    assert payload.pickle_protocol > 0
+    assert bundle.outputs[0].value_json_schema["title"] == "ContinuousScore"
+
+
+def test_cloudpickle_bundler_rejects_non_metric_object() -> None:
+    with pytest.raises(MetricBundlingError, match="Metric protocol"):
+        CloudpickleMetricBundler().bundle(_NotMetric())
+
+
+def test_cloudpickle_bundler_rejects_empty_metric_type() -> None:
+    with pytest.raises(MetricBundlingError, match="metric type must not be empty"):
+        CloudpickleMetricBundler().bundle(_EmptyTypeMetric())
+
+
+def test_cloudpickle_bundler_rejects_digest_mismatch() -> None:
+    bundler = CloudpickleMetricBundler()
+    bundle = bundler.bundle(_CustomMetric())
+    corrupted = bundle.model_copy(update={"digest": "0" * 64})
+
+    with pytest.raises(MetricBundlingError, match="digest"):
+        bundler.unbundle(corrupted)
diff --git a/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py b/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py
index 1cbd03af29..192e41dc1d 100644
--- a/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py
+++ b/packages/nemo_evaluator_sdk/tests/metrics/test_metric_contract.py
@@ -85,6 +85,11 @@ def test_metric_descriptor_rejects_duplicate_outputs() -> None:
         )
 
 
+def test_metric_descriptor_rejects_empty_metric_type() -> None:
+    with pytest.raises(ValueError, match="String should have at least 1 character"):
+        MetricDescriptor(type="", outputs=[MetricOutputSpec.continuous_score("score")])
+
+
 def test_validate_metric_result_accepts_declared_outputs() -> None:
     outputs = [
         MetricOutputSpec.continuous_score("reward"),
diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml
index 558d3315e2..d35483814b 100644
--- a/packages/nemo_platform/pyproject.toml
+++ b/packages/nemo_platform/pyproject.toml
@@ -119,6 +119,7 @@ entities-service = [
 
 # Generated from [tool.bundle-package]; do not edit by hand.
 evaluator = [
+  "cloudpickle>=3.1.1",
   "pydantic>=2.10.6",
   "jinja2>=3.1.6",
   "jsonschema>=4.23.0",
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py
new file mode 100644
index 0000000000..5991d96f96
--- /dev/null
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/compiler.py
@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Plugin-native evaluator job compiler."""
+
+from __future__ import annotations
+
+from nemo_evaluator.jobs.evaluate import EvaluateSpec
+from nemo_evaluator_sdk.metrics.base import MetricBundle
+from nemo_evaluator_sdk.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel
+from nemo_platform_plugin.jobs.api_factory import (
+    ContainerSpec,
+    CPUExecutionProviderSpec,
+    EnvironmentVariable,
+    EnvironmentVariableFromSecret,
+    PlatformJobSpec,
+    PlatformJobStep,
+)
+from nemo_platform_plugin.jobs.constants import (
+    DEFAULT_JOB_STORAGE_PATH,
+    EPHEMERAL_TASK_STORAGE_PATH_ENVVAR,
+    PERSISTENT_JOB_STORAGE_PATH_ENVVAR,
+)
+from nmp.common.jobs.image import get_qualified_image
+from nmp.evaluator.app.values import FilesetRef
+
+DATASET_DOWNLOAD_STEP_NAME = "dataset-download"
+EVALUATE_STEP_NAME = "evaluate"
+
+
+def compile_evaluate_job(spec: EvaluateSpec, *, profile: str | None = None) -> PlatformJobSpec:
+    """Compile a bundle-native evaluator plugin job."""
+    _validate_evaluate_spec(spec)
+    steps: list[PlatformJobStep] = []
+    if isinstance(spec.dataset, FilesetRef):
+        steps.append(_fileset_download_step(spec.dataset))
+    steps.append(_evaluate_step(spec, profile))
+    return PlatformJobSpec(steps=steps)
+
+
+def _validate_evaluate_spec(spec: EvaluateSpec) -> None:
+    if isinstance(spec.target, Model):
+        if spec.prompt_template is None:
+            raise ValueError("prompt_template is required when EvaluateSpec.target is a model")
+        if not isinstance(spec.params, RunConfigOnlineModel):
+            raise TypeError("model target requires RunConfigOnlineModel")
+    elif isinstance(spec.target, Agent):
+        if spec.prompt_template is None:
+            raise ValueError("prompt_template is required when EvaluateSpec.target is an agent")
+        if not isinstance(spec.params, RunConfigOnline):
+            raise TypeError("agent target requires RunConfigOnline")
+    elif not isinstance(spec.params, RunConfig):
+        raise TypeError("offline evaluation requires RunConfig")
+
+
+def _fileset_download_step(dataset: FilesetRef) -> PlatformJobStep:
+    scratch_path = "${" + EPHEMERAL_TASK_STORAGE_PATH_ENVVAR + "}"
+    target_download_dir = "${" + PERSISTENT_JOB_STORAGE_PATH_ENVVAR + "}/datasets"
+    return PlatformJobStep(
+        name=DATASET_DOWNLOAD_STEP_NAME,
+        executor=CPUExecutionProviderSpec(
+            provider="cpu",
+            container=ContainerSpec(
+                image=get_qualified_image("nmp-cpu-tasks"),
+                entrypoint=["python", "-m", "nmp.evaluator.tasks.download_fileset"],
+                command=[
+                    "--local-dir",
+                    scratch_path,
+                    "--target-dir",
+                    target_download_dir,
+                    "--dataset",
+                    dataset.model_dump_json(),
+                ],
+            ),
+        ),
+        environment=[
+            EnvironmentVariable(
+                name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR,
+                value=DEFAULT_JOB_STORAGE_PATH,
+            )
+        ],
+    )
+
+
+def _metric_bundles(spec: EvaluateSpec) -> list[MetricBundle]:
+    if isinstance(spec.metric, list):
+        return spec.metric
+    return [spec.metric]
+
+
+def _add_secret_ref(secret_refs: dict[str, str], env_name: str, secret_name: str) -> None:
+    existing = secret_refs.get(env_name)
+    if existing is not None and existing != secret_name:
+        raise ValueError(f"conflicting secret references for environment variable {env_name!r}")
+    secret_refs[env_name] = secret_name
+
+
+def _secret_environment(spec: EvaluateSpec) -> list[EnvironmentVariable]:
+    environment = [
+        EnvironmentVariable(
+            name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR,
+            value=DEFAULT_JOB_STORAGE_PATH,
+        )
+    ]
+    secret_refs: dict[str, str] = {}
+    for bundle in _metric_bundles(spec):
+        for env_name, secret_ref in bundle.secrets.items():
+            _add_secret_ref(secret_refs, env_name, secret_ref.root)
+
+    if isinstance(spec.target, Model | Agent) and spec.target.api_key_secret is not None and spec.target.api_key_env:
+        _add_secret_ref(secret_refs, spec.target.api_key_env, spec.target.api_key_secret.root)
+
+    environment.extend(
+        EnvironmentVariable(name=env_name, from_secret=EnvironmentVariableFromSecret(name=secret_name))
+        for env_name, secret_name in sorted(secret_refs.items())
+    )
+    return environment
+
+
+def _evaluate_step(spec: EvaluateSpec, profile: str | None) -> PlatformJobStep:
+    return PlatformJobStep(
+        name=EVALUATE_STEP_NAME,
+        executor=CPUExecutionProviderSpec(
+            profile=profile or "default",
+            provider="cpu",
+            container=ContainerSpec(
+                image=get_qualified_image("nmp-cpu-tasks"),
+                entrypoint=["python", "-m"],
+                command=["nemo_evaluator.tasks.evaluate"],
+            ),
+        ),
+        config=spec.model_dump(mode="json"),
+        environment=_secret_environment(spec),
+    )
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py
index aad98a1d80..bf7880c0aa 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py
@@ -6,13 +6,15 @@
 from __future__ import annotations
 
 import json
-from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
 from typing import Annotated, Any, ClassVar, Self, TypeAlias, cast
 
-from nemo_evaluator.jobs.utils import remote_compile_metric, resolve_run_dataset, resolve_submit_dataset
+from nemo_evaluator.jobs.utils import resolve_run_dataset
 from nemo_evaluator_sdk import Evaluator
 from nemo_evaluator_sdk.execution.config import normalize_params
-from nemo_evaluator_sdk.metrics.types import MetricsUnion
+from nemo_evaluator_sdk.metrics.base import MetricBundle, metric_bundler_for_payload
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricPayload  # noqa: F401
 from nemo_evaluator_sdk.values import (
     Agent,
     Model,
@@ -20,6 +22,7 @@
     RunConfigOnline,
     RunConfigOnlineModel,
 )
+from nemo_evaluator_sdk.values.results import EvaluationResult
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
 from nemo_platform_plugin.job import NemoJob
 from nemo_platform_plugin.job_context import JobContext
@@ -28,12 +31,28 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 TargetSpec = Model | Agent
-MetricSpec: TypeAlias = MetricsUnion | Annotated[Sequence[MetricsUnion], Field(min_length=1)]
+MetricSpec: TypeAlias = MetricBundle | Annotated[list[MetricBundle], Field(min_length=1)]
 InlineDataset: TypeAlias = Annotated[list[dict[str, object]], Field(min_length=1)]
 DatasetSpec: TypeAlias = InlineDataset | FilesetRef
 
 DEFAULT_RESULT_NAME = "evaluation-results"
 DEFAULT_FILE_NAME = "evaluation-results.json"
+ARTIFACTS_RESULT_NAME = "artifacts"
+AGGREGATE_SCORES_RESULT_NAME = "aggregate-scores"
+ROW_SCORES_RESULT_NAME = "row-scores"
+AGGREGATE_SCORES_FILE_NAME = "aggregate-scores.json"
+ROW_SCORES_FILE_NAME = "row-scores.jsonl"
+RESULT_IGNORE_PATTERNS = ["cache.db", "cache/"]
+
+
+@dataclass(frozen=True)
+class EvaluationResultFiles:
+    """Filesystem layout for an evaluator SDK result."""
+
+    full_result: Path
+    aggregate_scores: Path
+    row_scores: Path
+    artifacts_dir: Path
 
 
 class EvaluateSpec(BaseModel):
@@ -41,7 +60,7 @@ class EvaluateSpec(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    metric: MetricSpec = Field(description="Inline evaluator SDK metric configuration or benchmark metrics.")
+    metric: MetricSpec = Field(description="Bundled metric entity or benchmark metric bundle entities.")
     dataset: DatasetSpec = Field(
         description="Inline dataset rows or a persisted FilesetRef dataset source to evaluate.",
     )
@@ -80,52 +99,41 @@ async def compile(
         profile: str | None = None,
         options: dict | None = None,
     ) -> PlatformJobSpec:
-        """Compile canonical spec using the evaluator service metric job compiler."""
-        del workspace, entity_client, job_name, profile, options
+        """Compile canonical spec to a plugin-native evaluator job."""
+        del workspace, entity_client, job_name, async_sdk, options
+        from nemo_evaluator.jobs.compiler import compile_evaluate_job
+
         canonical_spec = spec if isinstance(spec, EvaluateSpec) else EvaluateSpec.model_validate(spec.model_dump())
+        return compile_evaluate_job(canonical_spec, profile=profile)
 
-        from nmp.evaluator.app.jobs.metrics import compile_metric_job
-        from nmp.evaluator.app.values import MetricOfflineJob, MetricOnlineAgentJob, MetricOnlineJob
-
-        dataset, dataset_ref = await resolve_submit_dataset(cast(AsyncNeMoPlatform, async_sdk), canonical_spec.dataset)
-        params = normalize_params(canonical_spec.params, canonical_spec.target)
-        metric = remote_compile_metric(canonical_spec.metric)
-        if isinstance(canonical_spec.target, Model):
-            if canonical_spec.prompt_template is None:
-                raise ValueError("prompt_template is required when EvaluateSpec.target is a model")
-            if not isinstance(params, RunConfigOnlineModel):
-                raise TypeError("model target requires RunConfigOnlineModel")
-            metric_job = MetricOnlineJob(
-                metric=metric,
-                model=canonical_spec.target,
-                dataset=dataset,
-                dataset_ref=dataset_ref,
-                params=params,
-                prompt_template=canonical_spec.prompt_template,
-            )
-        elif isinstance(canonical_spec.target, Agent):
-            if canonical_spec.prompt_template is None:
-                raise ValueError("prompt_template is required when EvaluateSpec.target is an agent")
-            if not isinstance(params, RunConfigOnline):
-                raise TypeError("agent target requires RunConfigOnline")
-            metric_job = MetricOnlineAgentJob(
-                metric=metric,
-                agent=canonical_spec.target,
-                dataset=dataset,
-                dataset_ref=dataset_ref,
-                params=params,
-                prompt_template=canonical_spec.prompt_template,
-            )
-        else:
-            if not isinstance(params, RunConfig):
-                raise TypeError("offline evaluation requires RunConfig")
-            metric_job = MetricOfflineJob(
-                metric=metric,
-                dataset=dataset,
-                dataset_ref=dataset_ref,
-                params=params,
-            )
-        return await compile_metric_job(metric_job)
+    @staticmethod
+    def _hydrate_metric(metric: MetricSpec):
+        if isinstance(metric, list):
+            return [metric_bundler_for_payload(bundle.payload).unbundle(bundle) for bundle in metric]
+        return metric_bundler_for_payload(metric.payload).unbundle(metric)
+
+    @staticmethod
+    def _write_result_files(result: EvaluationResult, persistent_dir: Path) -> EvaluationResultFiles:
+        """Write full, aggregate, and row-level evaluator artifacts."""
+        result_payload = result.model_dump(mode="json")
+        full_result_path = persistent_dir / DEFAULT_FILE_NAME
+        full_result_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8")
+
+        artifacts_dir = persistent_dir / ARTIFACTS_RESULT_NAME
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+        aggregate_path = artifacts_dir / AGGREGATE_SCORES_FILE_NAME
+        aggregate_path.write_text(result.aggregate_scores.model_dump_json(indent=2), encoding="utf-8")
+        row_scores_path = artifacts_dir / ROW_SCORES_FILE_NAME
+        with row_scores_path.open("w", encoding="utf-8") as f:
+            for row_score in result.row_scores:
+                f.write(row_score.model_dump_json() + "\n")
+
+        return EvaluationResultFiles(
+            full_result=full_result_path,
+            aggregate_scores=aggregate_path,
+            row_scores=row_scores_path,
+            artifacts_dir=artifacts_dir,
+        )
 
     def run(self, config: dict, *, ctx: JobContext, sdk: object | None = None, async_sdk: object | None = None) -> dict:
         """Run the evaluator job locally and persist its result artifact."""
@@ -143,14 +151,12 @@ def run(self, config: dict, *, ctx: JobContext, sdk: object | None = None, async
             "target": spec.target,
             "prompt_template": spec.prompt_template,
         }
-        if isinstance(spec.metric, Sequence):
-            result = evaluator.run_sync(metrics=spec.metric, **common_kwargs)
-        else:
-            result = evaluator.run_sync(metrics=cast(MetricsUnion, spec.metric), **common_kwargs)
-        result_payload = result.model_dump(mode="json")
-        result_path = ctx.storage.persistent / DEFAULT_FILE_NAME
-        result_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8")
-        artifact = ctx.results.save(DEFAULT_RESULT_NAME, result_path)
+        result = evaluator.run_sync(metrics=self._hydrate_metric(spec.metric), **common_kwargs)
+        result_files = self._write_result_files(result, ctx.storage.persistent)
+        artifact = ctx.results.save(DEFAULT_RESULT_NAME, result_files.full_result)
+        ctx.results.save(AGGREGATE_SCORES_RESULT_NAME, result_files.aggregate_scores)
+        ctx.results.save(ROW_SCORES_RESULT_NAME, result_files.row_scores)
+        ctx.results.save(ARTIFACTS_RESULT_NAME, result_files.artifacts_dir, ignore_patterns=RESULT_IGNORE_PATTERNS)
 
         # TODO: Implement progress reporting hook in SDK - AALGO-149
         # self.report_progress(
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py
index 13b8ea7f83..c17c7eb0c0 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py
@@ -5,41 +5,15 @@
 
 from __future__ import annotations
 
-from collections.abc import Sequence
-from typing import Any, cast
+from typing import Any
 
 from nemo_evaluator_sdk.execution.metric_execution import run_sync
-from nemo_evaluator_sdk.metrics.types import MetricsUnion
-from nemo_evaluator_sdk.values import DatasetRows
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
 from nemo_platform_plugin.job_context import JobContext
-from nmp.evaluator.app.datasets.nmp_datasets.fileset import dataset_exists, download_dataset, download_dataset_sync
+from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync
 from nmp.evaluator.app.values import FilesetRef
 
 
-def remote_compile_metric(metric: MetricsUnion | Sequence[MetricsUnion]) -> MetricsUnion:
-    """Return the single metric supported by remote service metric-job compilation."""
-    if isinstance(metric, Sequence):
-        raise NotImplementedError("Remote benchmark compilation is not implemented for inline evaluator plugin specs.")
-    return cast(MetricsUnion, metric)
-
-
-async def resolve_submit_dataset(
-    async_sdk: AsyncNeMoPlatform,
-    dataset: list[dict[str, object]] | FilesetRef,
-) -> tuple[DatasetRows | FilesetRef, FilesetRef | None]:
-    """Resolve an evaluator plugin dataset for remote metric-job submission.
-
-    FilesetRef datasets are validated via the async SDK and passed through;
-    inline rows are wrapped as ``DatasetRows``.
-    """
-    if isinstance(dataset, FilesetRef):
-        if not await dataset_exists(async_sdk, dataset):
-            raise ValueError(f"FilesetRef dataset does not exist: {dataset.root}")
-        return dataset, dataset
-    return DatasetRows(rows=dataset), None
-
-
 def resolve_run_dataset(
     dataset: list[dict[str, object]] | FilesetRef,
     *,
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py
index 86bb9f38e1..279f97d3e1 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py
@@ -7,12 +7,15 @@
 
 import asyncio
 from collections.abc import Sequence
-from typing import Any, Literal, Protocol, runtime_checkable
+from contextlib import asynccontextmanager, contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, AsyncIterator, Iterator, cast
 
 import httpx
 from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec
 from nemo_evaluator.sdk import http_utils
-from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult, local_result_path
+from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult
 from nemo_evaluator.sdk.job_resources import (
     AsyncEvaluatorJobResource,
     EvaluatorJob,
@@ -20,11 +23,14 @@
 )
 from nemo_evaluator.sdk.types import PluginDatasetInput
 from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result
+from nemo_evaluator_sdk import Evaluator as SDKEvaluator
 from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows
 from nemo_evaluator_sdk.execution.config import EvaluationRequest, normalize_params
+from nemo_evaluator_sdk.metrics.base import MetricBundle, MetricBundler
 from nemo_evaluator_sdk.metrics.protocol import Metric
 from nemo_evaluator_sdk.values import (
     Agent,
+    DatasetInput,
     Model,
     RunConfig,
     RunConfigOnline,
@@ -34,20 +40,27 @@
 from nemo_evaluator_sdk.values.results import AggregateFieldName, EvaluationResult
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
 from nemo_platform_plugin.scheduler import NemoJobScheduler
+from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync
 from nmp.evaluator.app.values import FilesetRef
 
 _DEFAULT_POLL_INTERVAL_SECONDS = 10.0
 _DEFAULT_JOB_TIMEOUT_SECONDS = 3600.0
 _DEFAULT_PENDING_TIMEOUT_SECONDS = 600.0
 
+_ResolvedDataset = DatasetInput | str | Path
 
-@runtime_checkable
-class _SerializableMetric(Protocol):
-    """Metric shape required for evaluator plugin job serialization."""
 
-    def model_dump(self, *, mode: Literal["json"]) -> dict[str, Any]:
-        """Return the metric as a JSON-serializable payload."""
-        ...
+class MetricBundlerPolicyError(RuntimeError):
+    """Raised when plugin backend metric bundling is not configured."""
+
+
+def _require_metric_bundler(metric_bundler: MetricBundler | None) -> MetricBundler:
+    if metric_bundler is None:
+        raise MetricBundlerPolicyError(
+            "Evaluator plugin remote execution requires an explicit metric_bundler. "
+            "Pass CloudpickleMetricBundler() to opt in to cloudpickle metric bundles."
+        )
+    return metric_bundler
 
 
 def _dataset_config(request: EvaluationRequest) -> list[dict[str, Any]] | FilesetRef:
@@ -65,10 +78,62 @@ def _dataset_config(request: EvaluationRequest) -> list[dict[str, Any]] | Filese
     )
 
 
-def _build_evaluate_spec(*, metrics: Metric | Sequence[Metric], request: EvaluationRequest) -> EvaluateSpec:
+def _fileset_dataset(request: EvaluationRequest) -> FilesetRef:
+    if not isinstance(request.dataset, FilesetRef):
+        raise TypeError("request dataset is not a FilesetRef")
+    if request.dataset_glob_pattern is None:
+        return request.dataset
+    if "#" in request.dataset.root:
+        raise ValueError("dataset_glob_pattern cannot be used when FilesetRef already includes a fragment.")
+    return request.dataset.with_fragment(request.dataset_glob_pattern)
+
+
+@contextmanager
+def _sync_resolved_dataset(
+    request: EvaluationRequest, platform: NeMoPlatform
+) -> Iterator[tuple[_ResolvedDataset, str | None]]:
+    if not isinstance(request.dataset, FilesetRef):
+        yield request.dataset, request.dataset_glob_pattern
+        return
+
+    dataset = _fileset_dataset(request)
+    with TemporaryDirectory(prefix="nemo-evaluator-fileset-") as temp_dir:
+        resolved = download_dataset_sync(
+            sdk=platform,
+            dataset=dataset,
+            destination=str(Path(temp_dir) / "dataset"),
+        )
+        yield cast(_ResolvedDataset, resolved), None
+
+
+@asynccontextmanager
+async def _async_resolved_dataset(
+    request: EvaluationRequest, platform: AsyncNeMoPlatform
+) -> AsyncIterator[tuple[_ResolvedDataset, str | None]]:
+    if not isinstance(request.dataset, FilesetRef):
+        yield request.dataset, request.dataset_glob_pattern
+        return
+
+    dataset = _fileset_dataset(request)
+    with TemporaryDirectory(prefix="nemo-evaluator-fileset-") as temp_dir:
+        resolved = await download_dataset(
+            sdk=platform,
+            dataset=dataset,
+            destination=str(Path(temp_dir) / "dataset"),
+        )
+        yield cast(_ResolvedDataset, resolved), None
+
+
+def _build_evaluate_spec(
+    *,
+    metrics: Metric | Sequence[Metric],
+    request: EvaluationRequest,
+    metric_bundler: MetricBundler | None = None,
+) -> EvaluateSpec:
     """Build the evaluator plugin spec shared by local and remote execution."""
+    effective_bundler = _require_metric_bundler(metric_bundler)
     spec = {
-        "metric": metrics_config(metrics),
+        "metric": bundle_metrics_for_spec(metrics, metric_bundler=effective_bundler),
         "dataset": _dataset_config(request),
         "params": request.params.model_dump(mode="json") if request.params else None,
     }
@@ -90,6 +155,7 @@ def __init__(
         poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS,
         job_timeout_seconds: float = _DEFAULT_JOB_TIMEOUT_SECONDS,
         pending_timeout_seconds: float = _DEFAULT_PENDING_TIMEOUT_SECONDS,
+        metric_bundler: MetricBundler | None = None,
     ) -> None:
         """Store the sync platform client used for evaluator execution."""
         self._platform = platform
@@ -98,6 +164,7 @@ def __init__(
         self._poll_interval_seconds = poll_interval_seconds
         self._job_timeout_seconds = job_timeout_seconds
         self._pending_timeout_seconds = pending_timeout_seconds
+        self._metric_bundler = metric_bundler
 
     def create(
         self,
@@ -150,9 +217,14 @@ def evaluate_remote(
         *,
         metric: Metric,
         request: EvaluationRequest,
+        metric_bundler: MetricBundler | None = None,
     ) -> EvaluationResult:
         """Submit, poll, and download a remote evaluator plugin metric job."""
-        spec = _build_evaluate_spec(metrics=metric, request=request)
+        spec = _build_evaluate_spec(
+            metrics=metric,
+            request=request,
+            metric_bundler=self._metric_bundler if metric_bundler is None else metric_bundler,
+        )
 
         job = self.create(
             spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True)
@@ -185,15 +257,15 @@ def evaluate(
             prompt_template=prompt_template,
             aggregate_fields=aggregate_fields,
         )
-        spec = _build_evaluate_spec(metrics=metric, request=request)
-
-        payload = self.run_local(
-            spec=spec,
-            workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True),
-        )
-
-        result_path = local_result_path(payload)
-        result = EvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8"))
+        with _sync_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern):
+            result = SDKEvaluator().run_sync(
+                metrics=metric,
+                dataset=resolved_dataset,
+                config=request.params,
+                target=request.target,
+                dataset_glob_pattern=resolved_pattern,
+                prompt_template=request.prompt_template,
+            )
         return filter_evaluation_result(result, aggregate_fields)
 
     def submit(
@@ -205,6 +277,7 @@ def submit(
         target: Model | Agent | None = None,
         dataset_glob_pattern: str | None = None,
         prompt_template: str | dict[str, Any] | None = None,
+        metric_bundler: MetricBundler | None = None,
     ) -> EvaluatorJobResource:
         """Submit a remote evaluator plugin metric job and return the job resource."""
         request = EvaluationRequest(
@@ -214,7 +287,11 @@ def submit(
             dataset_glob_pattern=dataset_glob_pattern,
             prompt_template=prompt_template,
         )
-        spec = _build_evaluate_spec(metrics=metric, request=request)
+        spec = _build_evaluate_spec(
+            metrics=metric,
+            request=request,
+            metric_bundler=self._metric_bundler if metric_bundler is None else metric_bundler,
+        )
 
         job = self.create(
             spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True)
@@ -229,15 +306,15 @@ def evaluate_benchmark(
         request: EvaluationRequest,
     ) -> BenchmarkEvaluationResult:
         """Evaluate multiple metrics through local in-process plugin execution."""
-        spec = _build_evaluate_spec(metrics=metrics, request=request)
-
-        payload = self.run_local(
-            spec=spec,
-            workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True),
-        )
-
-        result_path = local_result_path(payload)
-        result = BenchmarkEvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8"))
+        with _sync_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern):
+            result = SDKEvaluator().run_sync(
+                metrics=metrics,
+                dataset=resolved_dataset,
+                config=request.params,
+                target=request.target,
+                dataset_glob_pattern=resolved_pattern,
+                prompt_template=request.prompt_template,
+            )
         return filter_benchmark_result(result, request.aggregate_fields)
 
 
@@ -252,6 +329,7 @@ def __init__(
         poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS,
         job_timeout_seconds: float = _DEFAULT_JOB_TIMEOUT_SECONDS,
         pending_timeout_seconds: float = _DEFAULT_PENDING_TIMEOUT_SECONDS,
+        metric_bundler: MetricBundler | None = None,
     ) -> None:
         """Store the async platform client used for evaluator execution."""
         self._platform = platform
@@ -260,6 +338,7 @@ def __init__(
         self._poll_interval_seconds = poll_interval_seconds
         self._job_timeout_seconds = job_timeout_seconds
         self._pending_timeout_seconds = pending_timeout_seconds
+        self._metric_bundler = metric_bundler
 
     async def create(
         self,
@@ -319,6 +398,7 @@ async def submit(
         target: Model | Agent | None = None,
         dataset_glob_pattern: str | None = None,
         prompt_template: str | dict[str, Any] | None = None,
+        metric_bundler: MetricBundler | None = None,
     ) -> AsyncEvaluatorJobResource:
         """Submit a remote evaluator plugin metric job and return the job resource."""
         request = EvaluationRequest(
@@ -328,7 +408,11 @@ async def submit(
             dataset_glob_pattern=dataset_glob_pattern,
             prompt_template=prompt_template,
         )
-        spec = _build_evaluate_spec(metrics=metric, request=request)
+        spec = _build_evaluate_spec(
+            metrics=metric,
+            request=request,
+            metric_bundler=self._metric_bundler if metric_bundler is None else metric_bundler,
+        )
 
         job = await self.create(
             spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True)
@@ -341,9 +425,14 @@ async def evaluate_remote(
         *,
         metric: Metric,
         request: EvaluationRequest,
+        metric_bundler: MetricBundler | None = None,
     ) -> EvaluationResult:
         """Submit, poll, and download a remote evaluator plugin metric job."""
-        spec = _build_evaluate_spec(metrics=metric, request=request)
+        spec = _build_evaluate_spec(
+            metrics=metric,
+            request=request,
+            metric_bundler=self._metric_bundler if metric_bundler is None else metric_bundler,
+        )
 
         job = await self.create(
             spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True)
@@ -376,16 +465,15 @@ async def evaluate(
             prompt_template=prompt_template,
             aggregate_fields=aggregate_fields,
         )
-        spec = _build_evaluate_spec(metrics=metric, request=request)
-
-        payload = await self.run_local(
-            spec=spec,
-            workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True),
-        )
-
-        result_path = local_result_path(payload)
-        result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8")
-        result = EvaluationResult.model_validate_json(result_text)
+        async with _async_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern):
+            result = await SDKEvaluator().run(
+                metrics=metric,
+                dataset=resolved_dataset,
+                config=request.params,
+                target=request.target,
+                dataset_glob_pattern=resolved_pattern,
+                prompt_template=request.prompt_template,
+            )
         return filter_evaluation_result(result, aggregate_fields)
 
     async def evaluate_benchmark(
@@ -395,28 +483,23 @@ async def evaluate_benchmark(
         request: EvaluationRequest,
     ) -> BenchmarkEvaluationResult:
         """Evaluate multiple metrics through local in-process plugin execution."""
-        spec = _build_evaluate_spec(metrics=metrics, request=request)
-
-        payload = await self.run_local(
-            spec=spec,
-            workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True),
-        )
-
-        result_path = local_result_path(payload)
-        result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8")
-        result = BenchmarkEvaluationResult.model_validate_json(result_text)
+        async with _async_resolved_dataset(request, self._platform) as (resolved_dataset, resolved_pattern):
+            result = await SDKEvaluator().run(
+                metrics=metrics,
+                dataset=resolved_dataset,
+                config=request.params,
+                target=request.target,
+                dataset_glob_pattern=resolved_pattern,
+                prompt_template=request.prompt_template,
+            )
         return filter_benchmark_result(result, request.aggregate_fields)
 
 
-def metric_config(metric: object) -> dict[str, Any]:
-    """Serialize one metric config for evaluator plugin job submission."""
-    if not isinstance(metric, _SerializableMetric):
-        raise TypeError("metrics must provide model_dump(mode='json') for evaluator plugin execution")
-    return metric.model_dump(mode="json")
-
-
-def metrics_config(metrics: object | Sequence[object]) -> dict[str, Any] | list[dict[str, Any]]:
-    """Serialize one metric or a benchmark metric sequence for an evaluator plugin spec."""
+def bundle_metrics_for_spec(
+    metrics: Metric | Sequence[Metric], *, metric_bundler: MetricBundler
+) -> MetricBundle | list[MetricBundle]:
+    """Bundle one metric or a benchmark metric sequence for an evaluator plugin spec."""
     if isinstance(metrics, Sequence) and not isinstance(metrics, (str, bytes)):
-        return [metric_config(metric) for metric in metrics]
-    return metric_config(metrics)
+        metric_sequence = cast(Sequence[Metric], metrics)
+        return [metric_bundler.bundle(metric) for metric in metric_sequence]
+    return metric_bundler.bundle(cast(Metric, metrics))
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py
index 1339136d81..1d465cf542 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py
@@ -24,6 +24,7 @@
     RunConfigOnline,
     RunConfigOnlineModel,
 )
+from nemo_evaluator_sdk.metrics.base import MetricBundler
 from nemo_evaluator_sdk.metrics.protocol import Metric
 from nemo_evaluator_sdk.values import (
     Agent,
@@ -84,6 +85,7 @@ def submit(
         target: Model | Agent | None = None,
         dataset_glob_pattern: str | None = None,
         prompt_template: str | dict[str, Any] | None = None,
+        metric_bundler: MetricBundler | None = None,
     ) -> EvaluatorJobResource:
         """Submit a metric job through the evaluator plugin executor."""
         return self._executor.submit(
@@ -93,6 +95,7 @@ def submit(
             target=target,
             dataset_glob_pattern=dataset_glob_pattern,
             prompt_template=prompt_template,
+            metric_bundler=metric_bundler,
         )
 
     def run(
@@ -189,6 +192,7 @@ async def submit(
         target: Model | Agent | None = None,
         dataset_glob_pattern: str | None = None,
         prompt_template: str | dict[str, Any] | None = None,
+        metric_bundler: MetricBundler | None = None,
     ) -> AsyncEvaluatorJobResource:
         """Submit a metric job through the evaluator plugin executor."""
         return await self._executor.submit(
@@ -198,6 +202,7 @@ async def submit(
             target=target,
             dataset_glob_pattern=dataset_glob_pattern,
             prompt_template=prompt_template,
+            metric_bundler=metric_bundler,
         )
 
 
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py
index 43af13a789..4b7b0341de 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/standalone_sdk/backend.py
@@ -11,6 +11,7 @@
 from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator
 from nemo_evaluator.sdk.types import ExecutionMode
 from nemo_evaluator_sdk.execution.config import EvaluationRequest
+from nemo_evaluator_sdk.metrics.base import MetricBundler
 from nemo_evaluator_sdk.metrics.protocol import Metric
 from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult
 from nemo_evaluator_sdk.values.results import EvaluationResult
@@ -30,6 +31,7 @@ class NMPBackend:
 
     resource: Evaluator
     execution_mode: ExecutionMode = "local"
+    metric_bundler: MetricBundler | None = None
 
     def evaluate(
         self,
@@ -40,7 +42,11 @@ def evaluate(
         """Evaluate one metric through local or remote evaluator plugin execution."""
         _reject_unsupported_hooks(request)
         if self.execution_mode == "remote":
-            return self.resource._executor.evaluate_remote(metric=metric, request=request)
+            return self.resource._executor.evaluate_remote(
+                metric=metric,
+                request=request,
+                metric_bundler=self.metric_bundler,
+            )
         return self.resource._executor.evaluate(
             metric=metric,
             dataset=request.dataset,
@@ -61,7 +67,10 @@ def evaluate_benchmark(
         _reject_unsupported_hooks(request)
         if self.execution_mode == "remote":
             raise NotImplementedError("Remote evaluation of benchmarks is not implemented yet.")
-        return self.resource._executor.evaluate_benchmark(metrics=metrics, request=request)
+        return self.resource._executor.evaluate_benchmark(
+            metrics=metrics,
+            request=request,
+        )
 
 
 @dataclass(frozen=True, slots=True)
@@ -72,6 +81,7 @@ class AsyncNMPBackend:
 
     resource: AsyncEvaluator
     execution_mode: ExecutionMode = "local"
+    metric_bundler: MetricBundler | None = None
 
     async def evaluate(
         self,
@@ -82,7 +92,11 @@ async def evaluate(
         """Evaluate one metric through local or remote evaluator plugin execution."""
         _reject_unsupported_hooks(request)
         if self.execution_mode == "remote":
-            return await self.resource._executor.evaluate_remote(metric=metric, request=request)
+            return await self.resource._executor.evaluate_remote(
+                metric=metric,
+                request=request,
+                metric_bundler=self.metric_bundler,
+            )
         return await self.resource._executor.evaluate(
             metric=metric,
             dataset=request.dataset,
@@ -103,4 +117,7 @@ async def evaluate_benchmark(
         _reject_unsupported_hooks(request)
         if self.execution_mode == "remote":
             raise NotImplementedError("Remote evaluation of benchmarks is not implemented yet.")
-        return await self.resource._executor.evaluate_benchmark(metrics=metrics, request=request)
+        return await self.resource._executor.evaluate_benchmark(
+            metrics=metrics,
+            request=request,
+        )
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py
new file mode 100644
index 0000000000..8bacacd2a2
--- /dev/null
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py
@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Container entrypoint for evaluator plugin bundle-native jobs."""
+
+from __future__ import annotations
+
+from nemo_evaluator.jobs.evaluate import EvaluateJob
+from nemo_platform_plugin.tasks.dispatcher import run_task
+from nmp.common.sdk_factory import get_task_sdk
+
+
+def main() -> int:
+    """Run the evaluator job in a platform-spawned task process."""
+    return run_task(EvaluateJob, sdk=get_task_sdk("evaluator"))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py
index fd0bfe18b8..6aa27996d7 100644
--- a/plugins/nemo-evaluator/tests/test_evaluate_job.py
+++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py
@@ -11,10 +11,33 @@
 
 import pytest
 from nemo_evaluator.cli import EvaluatorPluginCLI
-from nemo_evaluator.jobs.evaluate import DEFAULT_FILE_NAME, DEFAULT_RESULT_NAME, EvaluateJob, EvaluateSpec
+from nemo_evaluator.jobs.evaluate import (
+    AGGREGATE_SCORES_RESULT_NAME,
+    ARTIFACTS_RESULT_NAME,
+    DEFAULT_FILE_NAME,
+    DEFAULT_RESULT_NAME,
+    ROW_SCORES_RESULT_NAME,
+    EvaluateJob,
+    EvaluateSpec,
+)
+from nemo_evaluator.tasks.evaluate import main as evaluate_task_main
 from nemo_evaluator_sdk.enums import AgentFormat
+from nemo_evaluator_sdk.metrics.base import MetricBundle
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler
+from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+from nemo_evaluator_sdk.metrics.f1 import F1Metric
 from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
-from nemo_evaluator_sdk.values import Agent, Model, RunConfig, RunConfigOnline, RunConfigOnlineModel
+from nemo_evaluator_sdk.values import (
+    Agent,
+    AggregatedMetricResult,
+    EvaluationResult,
+    Model,
+    RunConfig,
+    RunConfigOnline,
+    RunConfigOnlineModel,
+    SecretRef,
+)
+from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore
 from nemo_platform.types.jobs.platform_job_spec import PlatformJobSpec
 from nemo_platform_plugin.commands import add_job_commands
 from nemo_platform_plugin.job_context import JobContext, StoragePaths
@@ -28,11 +51,7 @@
 
 def _exact_match_spec() -> dict:
     return {
-        "metric": {
-            "type": "exact-match",
-            "reference": "{{item.expected}}",
-            "candidate": "{{item.model_output}}",
-        },
+        "metric": _bundle_payload(ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}")),
         "dataset": [
             {"expected": "blue", "model_output": "Blue"},
             {"expected": "Jupiter", "model_output": "Saturn"},
@@ -41,16 +60,15 @@ def _exact_match_spec() -> dict:
     }
 
 
+def _bundle_payload(metric) -> dict[str, Any]:
+    return CloudpickleMetricBundler().bundle(metric).model_dump(mode="json")
+
+
 def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None:
     step = job_spec.steps[0]
     container = cast(Any, step.executor).container
-    assert container.entrypoint == ["python", "-m", "nmp.evaluator.tasks.evaluate_metric"]
-    command = container.command
-    assert command is not None
-    assert command == [
-        "--progress-tracking-url",
-        "${NMP_JOBS_URL}/apis/jobs/v2/workspaces/${NEMO_JOB_WORKSPACE}/jobs/${NEMO_JOB_ID}/status-details",
-    ]
+    assert container.entrypoint == ["python", "-m"]
+    assert container.command == ["nemo_evaluator.tasks.evaluate"]
 
 
 def _load_cli_run_payload(output: str) -> dict[str, Any]:
@@ -70,6 +88,11 @@ def _make_job_context(tmp_path: Path) -> JobContext:
     )
 
 
+def _empty_evaluation_result() -> EvaluationResult:
+    """Return an SDK result object suitable for runner delegation tests."""
+    return EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
+
+
 def _assert_saved_result_artifact(
     run_result: dict[str, Any], ctx: JobContext, result_payload: dict[str, object]
 ) -> None:
@@ -82,6 +105,9 @@ def _assert_saved_result_artifact(
     assert json.loads(result_path.read_text(encoding="utf-8")) == result_payload
     artifact_path = Path(run_result["artifact"]["artifact_url"].removeprefix("file://"))
     assert json.loads(artifact_path.read_text(encoding="utf-8")) == result_payload
+    assert (ctx.storage.persistent / "results" / AGGREGATE_SCORES_RESULT_NAME).exists()
+    assert (ctx.storage.persistent / "results" / ROW_SCORES_RESULT_NAME).exists()
+    assert (ctx.storage.persistent / "results" / ARTIFACTS_RESULT_NAME).is_dir()
 
 
 def _load_artifact_payload(run_result: dict[str, Any]) -> dict[str, Any]:
@@ -149,12 +175,13 @@ async def test_evaluate_job_compile_produces_cpu_task_step() -> None:
     job_spec = PlatformJobSpec.model_validate(compiled)
     assert len(job_spec.steps) == 1
     step = job_spec.steps[0]
-    assert step.name == "evaluation"
+    assert step.name == "evaluate"
     _assert_metric_step_entrypoint(job_spec)
     assert step.config is not None
     config = cast(dict[str, Any], step.config)
-    assert config["metric"]["type"] == "exact-match"
-    assert config["dataset"]["rows"] == _exact_match_spec()["dataset"]
+    assert config["metric"]["bundle_kind"] == "metric-bundle"
+    assert config["metric"]["metric_type"] == "exact-match"
+    assert config["dataset"] == _exact_match_spec()["dataset"]
 
 
 async def test_evaluate_job_compile_produces_online_model_job() -> None:
@@ -179,7 +206,7 @@ async def test_evaluate_job_compile_produces_online_model_job() -> None:
     step = job_spec.steps[0]
     config = cast(dict[str, Any], step.config)
     _assert_metric_step_entrypoint(job_spec)
-    assert config["model"]["name"] == "test-model"
+    assert config["target"]["name"] == "test-model"
     assert config["prompt_template"] == "Question: {{item.question}}"
     assert config["params"]["parallelism"] == 3
 
@@ -211,10 +238,55 @@ async def test_evaluate_job_compile_produces_online_agent_job() -> None:
     step = job_spec.steps[0]
     config = cast(dict[str, Any], step.config)
     _assert_metric_step_entrypoint(job_spec)
-    assert config["agent"]["name"] == "test-agent"
+    assert config["target"]["name"] == "test-agent"
     assert config["prompt_template"] == {"question": "{{item.question}}"}
 
 
+async def test_evaluate_job_compile_injects_metric_and_target_secrets() -> None:
+    secret_ref = SecretRef(root="NVIDIA_BUILD_API_KEY")
+    spec = EvaluateSpec.model_validate(
+        {
+            **_exact_match_spec(),
+            "metric": _bundle_payload(
+                LLMJudgeMetric(
+                    model=Model(
+                        url="https://integrate.api.nvidia.com/v1/chat/completions",
+                        name="nvidia/nemotron-3-super-120b-a12b",
+                        api_key_secret=secret_ref,
+                    ),
+                    scores=[
+                        RangeScore(
+                            name="quality",
+                            minimum=1,
+                            maximum=5,
+                            parser=JSONScoreParser(json_path="quality"),
+                        ),
+                    ],
+                )
+            ),
+            "target": Model(
+                url="https://integrate.api.nvidia.com/v1/chat/completions",
+                name="nvidia/nemotron-3-super-120b-a12b",
+                api_key_secret=secret_ref,
+            ),
+            "params": RunConfigOnlineModel(parallelism=3),
+            "prompt_template": "Question: {{item.question}}",
+        }
+    )
+
+    compiled = await EvaluateJob.compile(
+        workspace="default",
+        spec=spec,
+        entity_client=object(),
+        job_name=None,
+        async_sdk=object(),
+    )
+
+    step = PlatformJobSpec.model_validate(compiled).steps[0]
+    secrets = {env.name: env.from_secret.name for env in step.environment or [] if env.from_secret}
+    assert secrets == {"NVIDIA_BUILD_API_KEY": "NVIDIA_BUILD_API_KEY"}
+
+
 class TestEvaluateSpec:
     """Validation coverage for evaluator job specs."""
 
@@ -246,38 +318,34 @@ def test_accepts_metrics_sequence(self) -> None:
                 **_exact_match_spec(),
                 "metric": [
                     _exact_match_spec()["metric"],
-                    {
-                        "type": "f1",
-                        "reference": "{{item.expected}}",
-                        "candidate": "{{item.model_output}}",
-                    },
+                    _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")),
                 ],
             }
         )
 
         assert isinstance(spec.metric, list)
-        assert [metric.type.value for metric in spec.metric] == ["exact-match", "f1"]
+        assert [metric.metric_type for metric in spec.metric] == ["exact-match", "f1"]
 
     def test_accepts_uppercase_api_key_secret_refs_for_llm_judge_and_target(self) -> None:
         spec = EvaluateSpec.model_validate(
             {
-                "metric": {
-                    "type": "llm-judge",
-                    "model": {
-                        "url": "https://integrate.api.nvidia.com/v1/chat/completions",
-                        "name": "nvidia/nemotron-3-super-120b-a12b",
-                        "api_key_secret": "NVIDIA_BUILD_API_KEY",
-                        "format": "nim",
-                    },
-                    "scores": [
-                        {
-                            "name": "quality",
-                            "minimum": 1,
-                            "maximum": 5,
-                            "parser": {"type": "json", "json_path": "quality"},
-                        },
-                    ],
-                },
+                "metric": _bundle_payload(
+                    LLMJudgeMetric(
+                        model=Model(
+                            url="https://integrate.api.nvidia.com/v1/chat/completions",
+                            name="nvidia/nemotron-3-super-120b-a12b",
+                            api_key_secret=SecretRef(root="NVIDIA_BUILD_API_KEY"),
+                        ),
+                        scores=[
+                            RangeScore(
+                                name="quality",
+                                minimum=1,
+                                maximum=5,
+                                parser=JSONScoreParser(json_path="quality"),
+                            ),
+                        ],
+                    )
+                ),
                 "dataset": [{"prompt": "Hello", "model_output": "Hi"}],
                 "target": {
                     "url": "https://integrate.api.nvidia.com/v1/chat/completions",
@@ -288,11 +356,10 @@ def test_accepts_uppercase_api_key_secret_refs_for_llm_judge_and_target(self) ->
             }
         )
 
-        assert isinstance(spec.metric, LLMJudgeMetric)
+        assert isinstance(spec.metric, MetricBundle)
         assert isinstance(spec.target, Model)
-        assert spec.metric.model.api_key_secret is not None
         assert spec.target.api_key_secret is not None
-        assert spec.metric.model.api_key_secret.root == "NVIDIA_BUILD_API_KEY"
+        assert spec.metric.metric_type == "llm-judge"
         assert spec.target.api_key_secret.root == "NVIDIA_BUILD_API_KEY"
 
     def test_rejects_extra_fields(self) -> None:
@@ -346,33 +413,32 @@ class EquivalentSpec(BaseModel):
         job_spec = PlatformJobSpec.model_validate(compiled)
         step = job_spec.steps[0]
         config = cast(dict[str, Any], step.config)
-        assert config["metric"]["type"] == "exact-match"
-        assert config["dataset"]["rows"] == _exact_match_spec()["dataset"]
+        assert config["metric"]["bundle_kind"] == "metric-bundle"
+        assert config["metric"]["metric_type"] == "exact-match"
+        assert config["dataset"] == _exact_match_spec()["dataset"]
         assert config["params"]["parallelism"] == 2
 
-    async def test_rejects_remote_compile_for_metrics_sequence(self) -> None:
+    async def test_accepts_metrics_sequence(self) -> None:
         spec = EvaluateSpec.model_validate(
             {
                 **_exact_match_spec(),
                 "metric": [
                     _exact_match_spec()["metric"],
-                    {
-                        "type": "f1",
-                        "reference": "{{item.expected}}",
-                        "candidate": "{{item.model_output}}",
-                    },
+                    _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")),
                 ],
             }
         )
 
-        with pytest.raises(NotImplementedError, match="Remote benchmark.*not implemented"):
-            await EvaluateJob.compile(
-                workspace="default",
-                spec=spec,
-                entity_client=object(),
-                job_name=None,
-                async_sdk=object(),
-            )
+        compiled = await EvaluateJob.compile(
+            workspace="default",
+            spec=spec,
+            entity_client=object(),
+            job_name=None,
+            async_sdk=object(),
+        )
+
+        config = cast(dict[str, Any], PlatformJobSpec.model_validate(compiled).steps[0].config)
+        assert [metric["metric_type"] for metric in config["metric"]] == ["exact-match", "f1"]
 
     @pytest.mark.parametrize(
         ("target", "expected_message"),
@@ -461,49 +527,25 @@ async def test_rejects_wrong_offline_param_type(self, mocker: MockerFixture) ->
                 async_sdk=object(),
             )
 
-    async def test_fileset_ref_dataset_validates_and_compiles_with_download_step(self, mocker: MockerFixture) -> None:
+    async def test_fileset_ref_dataset_compiles_into_bundle_native_step(self) -> None:
         dataset = FilesetRef(root="default/helpsteer2#validation/*.jsonl")
-        dataset_exists = mocker.patch(
-            "nemo_evaluator.jobs.utils.dataset_exists",
-            new=mocker.AsyncMock(return_value=True),
-            create=True,
-        )
-        async_sdk = object()
 
         compiled = await EvaluateJob.compile(
             workspace="default",
             spec=EvaluateSpec.model_validate({**_exact_match_spec(), "dataset": dataset}),
             entity_client=object(),
             job_name=None,
-            async_sdk=async_sdk,
+            async_sdk=object(),
         )
 
         job_spec = PlatformJobSpec.model_validate(compiled)
-        assert [step.name for step in job_spec.steps] == ["dataset-download", "evaluation"]
+        assert [step.name for step in job_spec.steps] == ["dataset-download", "evaluate"]
+        download_step = job_spec.steps[0]
+        download_container = cast(Any, download_step.executor).container
+        assert download_container.entrypoint == ["python", "-m", "nmp.evaluator.tasks.download_fileset"]
+        assert download_container.command[-2:] == ["--dataset", dataset.model_dump_json()]
         config = cast(dict[str, Any], job_spec.steps[1].config)
         assert config["dataset"] == dataset.root
-        assert config["dataset_ref"] == dataset.root
-        dataset_exists.assert_awaited_once_with(async_sdk, dataset)
-
-    async def test_fileset_ref_dataset_compile_raises_when_dataset_does_not_exist(self, mocker: MockerFixture) -> None:
-        dataset = FilesetRef(root="default/missing")
-        dataset_exists = mocker.patch(
-            "nemo_evaluator.jobs.utils.dataset_exists",
-            new=mocker.AsyncMock(return_value=False),
-            create=True,
-        )
-        async_sdk = object()
-
-        with pytest.raises(ValueError, match="FilesetRef dataset does not exist: default/missing"):
-            await EvaluateJob.compile(
-                workspace="default",
-                spec=EvaluateSpec.model_validate({**_exact_match_spec(), "dataset": dataset}),
-                entity_client=object(),
-                job_name=None,
-                async_sdk=async_sdk,
-            )
-
-        dataset_exists.assert_awaited_once_with(async_sdk, dataset)
 
 
 class TestEvaluateJobRun:
@@ -542,9 +584,8 @@ def test_delegates_to_sdk_evaluator(
         tmp_path: Path,
         mocker: MockerFixture,
     ) -> None:
-        result_payload = {"aggregate_scores": {"scores": []}}
-        result = mocker.Mock()
-        result.model_dump.return_value = result_payload
+        result = _empty_evaluation_result()
+        result_payload = result.model_dump(mode="json")
         evaluator = mocker.Mock()
         evaluator.run_sync.return_value = result
         evaluator_cls = mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator)
@@ -566,23 +607,16 @@ def test_delegates_to_sdk_evaluator(
         assert "result" not in run_result
         _assert_saved_result_artifact(run_result, ctx, result_payload)
         evaluator_cls.assert_called_once_with()
-        evaluator.run_sync.assert_called_once_with(
-            metrics=expected_spec.metric,
-            dataset=expected_spec.dataset,
-            config=expected_config,
-            target=expected_spec.target,
-            prompt_template=expected_spec.prompt_template,
-        )
-        result.model_dump.assert_called_once_with(mode="json")
+        call_kwargs = evaluator.run_sync.call_args.kwargs
+        assert isinstance(call_kwargs["metrics"], ExactMatchMetric)
+        assert call_kwargs["dataset"] == expected_spec.dataset
+        assert call_kwargs["config"] == expected_config
+        assert call_kwargs["target"] == expected_spec.target
+        assert call_kwargs["prompt_template"] == expected_spec.prompt_template
 
     def test_delegates_metrics_sequence_to_sdk_evaluator(self, tmp_path: Path, mocker: MockerFixture) -> None:
-        result_payload = {
-            "row_scores": [],
-            "aggregate_scores": {"scores": []},
-            "per_metric": {},
-        }
-        result = mocker.Mock()
-        result.model_dump.return_value = result_payload
+        result = _empty_evaluation_result()
+        result_payload = result.model_dump(mode="json")
         evaluator = mocker.Mock()
         evaluator.run_sync.return_value = result
         evaluator_cls = mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator)
@@ -590,11 +624,7 @@ def test_delegates_metrics_sequence_to_sdk_evaluator(self, tmp_path: Path, mocke
             **_exact_match_spec(),
             "metric": [
                 _exact_match_spec()["metric"],
-                {
-                    "type": "f1",
-                    "reference": "{{item.expected}}",
-                    "candidate": "{{item.model_output}}",
-                },
+                _bundle_payload(F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}")),
             ],
         }
         expected_spec = EvaluateSpec.model_validate(config)
@@ -609,21 +639,18 @@ def test_delegates_metrics_sequence_to_sdk_evaluator(self, tmp_path: Path, mocke
         assert "result" not in run_result
         _assert_saved_result_artifact(run_result, ctx, result_payload)
         evaluator_cls.assert_called_once_with()
-        evaluator.run_sync.assert_called_once_with(
-            metrics=expected_spec.metric,
-            dataset=expected_spec.dataset,
-            config=expected_spec.params,
-            target=expected_spec.target,
-            prompt_template=expected_spec.prompt_template,
-        )
-        result.model_dump.assert_called_once_with(mode="json")
+        call_kwargs = evaluator.run_sync.call_args.kwargs
+        assert [metric.type.value for metric in call_kwargs["metrics"]] == ["exact-match", "f1"]
+        assert call_kwargs["dataset"] == expected_spec.dataset
+        assert call_kwargs["config"] == expected_spec.params
+        assert call_kwargs["target"] == expected_spec.target
+        assert call_kwargs["prompt_template"] == expected_spec.prompt_template
 
     def test_downloads_fileset_ref_dataset_and_passes_path_to_sdk_evaluator(
         self, tmp_path: Path, mocker: MockerFixture
     ) -> None:
-        result_payload = {"aggregate_scores": {"scores": []}}
-        result = mocker.Mock()
-        result.model_dump.return_value = result_payload
+        result = _empty_evaluation_result()
+        result_payload = result.model_dump(mode="json")
         evaluator = mocker.Mock()
         evaluator.run_sync.return_value = result
         mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator)
@@ -648,21 +675,18 @@ def test_downloads_fileset_ref_dataset_and_passes_path_to_sdk_evaluator(
             destination=str(ctx.storage.persistent / "dataset"),
         )
         download_dataset_sync.assert_not_called()
-        evaluator.run_sync.assert_called_once_with(
-            metrics=EvaluateSpec.model_validate(config).metric,
-            dataset=downloaded_path,
-            config=EvaluateSpec.model_validate(config).params,
-            target=None,
-            prompt_template=None,
-        )
-        result.model_dump.assert_called_once_with(mode="json")
+        call_kwargs = evaluator.run_sync.call_args.kwargs
+        assert isinstance(call_kwargs["metrics"], ExactMatchMetric)
+        assert call_kwargs["dataset"] == downloaded_path
+        assert call_kwargs["config"] == EvaluateSpec.model_validate(config).params
+        assert call_kwargs["target"] is None
+        assert call_kwargs["prompt_template"] is None
 
     def test_downloads_fileset_ref_dataset_with_sync_sdk_and_passes_path_to_sdk_evaluator(
         self, tmp_path: Path, mocker: MockerFixture
     ) -> None:
-        result_payload = {"aggregate_scores": {"scores": []}}
-        result = mocker.Mock()
-        result.model_dump.return_value = result_payload
+        result = _empty_evaluation_result()
+        result_payload = result.model_dump(mode="json")
         evaluator = mocker.Mock()
         evaluator.run_sync.return_value = result
         mocker.patch("nemo_evaluator.jobs.evaluate.Evaluator", return_value=evaluator)
@@ -687,11 +711,24 @@ def test_downloads_fileset_ref_dataset_with_sync_sdk_and_passes_path_to_sdk_eval
             dataset=dataset,
             destination=str(ctx.storage.persistent / "dataset"),
         )
-        evaluator.run_sync.assert_called_once_with(
-            metrics=EvaluateSpec.model_validate(config).metric,
-            dataset=downloaded_path,
-            config=EvaluateSpec.model_validate(config).params,
-            target=None,
-            prompt_template=None,
-        )
-        result.model_dump.assert_called_once_with(mode="json")
+        call_kwargs = evaluator.run_sync.call_args.kwargs
+        assert isinstance(call_kwargs["metrics"], ExactMatchMetric)
+        assert call_kwargs["dataset"] == downloaded_path
+        assert call_kwargs["config"] == EvaluateSpec.model_validate(config).params
+        assert call_kwargs["target"] is None
+        assert call_kwargs["prompt_template"] is None
+
+
+class TestEvaluateTask:
+    """Coverage for the compiled container task entrypoint."""
+
+    def test_main_dispatches_evaluate_job_with_task_sdk(self, mocker: MockerFixture) -> None:
+        sdk = object()
+        get_task_sdk = mocker.patch("nemo_evaluator.tasks.evaluate.get_task_sdk", return_value=sdk)
+        run_task = mocker.patch("nemo_evaluator.tasks.evaluate.run_task", return_value=0)
+
+        exit_code = evaluate_task_main()
+
+        assert exit_code == 0
+        get_task_sdk.assert_called_once_with("evaluator")
+        run_task.assert_called_once_with(EvaluateJob, sdk=sdk)
diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py
index 401bac080c..55ea3e9546 100644
--- a/plugins/nemo-evaluator/tests/test_sdk.py
+++ b/plugins/nemo-evaluator/tests/test_sdk.py
@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
+from pathlib import Path
 from typing import Any, cast
 from unittest.mock import AsyncMock, MagicMock
 
@@ -14,18 +15,19 @@
 from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec
 from nemo_evaluator.sdk import http_utils
 from nemo_evaluator.sdk._executor import (
+    MetricBundlerPolicyError,
     _AsyncEvaluatorPluginExecutor,
     _build_evaluate_spec,
     _SyncEvaluatorPluginExecutor,
-    metric_config,
+    bundle_metrics_for_spec,
 )
 from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult
 from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource
 from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator
-from nemo_evaluator_sdk.enums import MetricType
 from nemo_evaluator_sdk.execution.config import EvaluationRequest
+from nemo_evaluator_sdk.metrics.base import MetricBundle, MetricBundlingError
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler
 from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
-from nemo_evaluator_sdk.metrics.types import MetricsUnion
 from nemo_evaluator_sdk.values import Model, RunConfig, RunConfigOnlineModel
 from nemo_evaluator_sdk.values.results import AggregatedMetricResult, EvaluationResult
 from nemo_platform import AsyncNeMoPlatform, NeMoPlatform
@@ -34,7 +36,12 @@
 from pydantic import ValidationError
 from pytest_mock import MockerFixture
 
+_EXACT_MATCH_METRIC = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
 _EXACT_MATCH_SPEC = {
+    "metric": CloudpickleMetricBundler().bundle(_EXACT_MATCH_METRIC).model_dump(mode="json"),
+    "dataset": [{"expected": "a", "output": "a"}],
+}
+_LEGACY_EXACT_MATCH_SPEC = {
     "metric": {
         "type": "exact-match",
         "reference": "{{item.expected}}",
@@ -46,7 +53,7 @@
 _EXACT_MATCH_EVALUATE_SPEC_JSON = _EXACT_MATCH_EVALUATE_SPEC.model_dump(mode="json")
 
 
-def _single_metric(spec: EvaluateSpec) -> MetricsUnion:
+def _single_metric(spec: EvaluateSpec) -> MetricBundle:
     """Return the single metric from an evaluator job spec."""
     if isinstance(spec.metric, Sequence):
         raise AssertionError("Expected a single metric spec.")
@@ -150,10 +157,18 @@ def test_resolve_workspace_requires_explicit_or_default_workspace() -> None:
         http_utils.resolve_workspace(cast(NeMoPlatform, _PlatformWithoutWorkspace()), None, strict=True)
 
 
-def test_metric_config_rejects_non_serializable_metric() -> None:
-    """Metrics must expose a JSON model dump for evaluator plugin execution."""
-    with pytest.raises(TypeError, match="model_dump"):
-        metric_config(object())
+def test_bundle_metrics_for_spec_rejects_non_metric_object() -> None:
+    """Metrics must satisfy the runtime Metric protocol before plugin execution."""
+    with pytest.raises(MetricBundlingError, match="Metric protocol"):
+        bundle_metrics_for_spec(object(), metric_bundler=CloudpickleMetricBundler())
+
+
+def test_build_evaluate_spec_requires_metric_bundler() -> None:
+    with pytest.raises(MetricBundlerPolicyError, match="CloudpickleMetricBundler"):
+        _build_evaluate_spec(
+            metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+            request=EvaluationRequest(dataset=[{"expected": "a", "output": "a"}]),
+        )
 
 
 def test_build_evaluate_spec_includes_target_and_prompt_template() -> None:
@@ -161,6 +176,7 @@ def test_build_evaluate_spec_includes_target_and_prompt_template() -> None:
     model = Model(url="https://model.test/v1", name="model-a")
     spec = _build_evaluate_spec(
         metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+        metric_bundler=CloudpickleMetricBundler(),
         request=EvaluationRequest(
             dataset=[{"expected": "a", "output": "a"}],
             target=model,
@@ -176,6 +192,7 @@ def test_build_evaluate_spec_excludes_aggregate_fields() -> None:
     """Evaluator specs should not persist result-shaping options."""
     spec = _build_evaluate_spec(
         metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+        metric_bundler=CloudpickleMetricBundler(),
         request=EvaluationRequest(
             dataset=[{"expected": "a", "output": "a"}],
             params=RunConfig(),
@@ -193,6 +210,7 @@ def test_build_evaluate_spec_preserves_fileset_ref_dataset() -> None:
 
     spec = _build_evaluate_spec(
         metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+        metric_bundler=CloudpickleMetricBundler(),
         request=EvaluationRequest(dataset=cast(Any, dataset)),
     )
 
@@ -203,6 +221,7 @@ def test_build_evaluate_spec_synthesizes_fileset_ref_fragment_from_dataset_glob_
     """FilesetRef datasets should encode dataset_glob_pattern as the existing fragment selector syntax."""
     spec = _build_evaluate_spec(
         metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+        metric_bundler=CloudpickleMetricBundler(),
         request=EvaluationRequest(
             dataset=cast(Any, FilesetRef(root="default/helpsteer2")),
             dataset_glob_pattern="validation/*.jsonl",
@@ -217,6 +236,7 @@ def test_build_evaluate_spec_rejects_fileset_ref_fragment_and_dataset_glob_patte
     with pytest.raises(ValueError, match=r"dataset_glob_pattern.*FilesetRef"):
         _build_evaluate_spec(
             metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"),
+            metric_bundler=CloudpickleMetricBundler(),
             request=EvaluationRequest(
                 dataset=cast(Any, FilesetRef(root="default/helpsteer2#validation/*.jsonl")),
                 dataset_glob_pattern="train/*.jsonl",
@@ -277,7 +297,7 @@ def test_sync_executor_creates_evaluator_job() -> None:
     assert job.name == "job-123"
     assert job.job.status == PlatformJobStatus.CREATED
     assert job.job.spec is not None
-    assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH
+    assert _single_metric(job.job.spec).metric_type == "exact-match"
     platform._client.post.assert_called_once_with(
         "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs",
         json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON},
@@ -315,7 +335,7 @@ def test_sync_executor_create_uses_platform_workspace_by_default() -> None:
     job = executor.create(spec=_EXACT_MATCH_EVALUATE_SPEC)
     assert job.name == "job-123"
     assert job.job.spec is not None
-    assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH
+    assert _single_metric(job.job.spec).metric_type == "exact-match"
     platform._client.post.assert_called_once_with(
         "http://test:8000/apis/evaluator/v2/workspaces/platform-ws/evaluate/jobs",
         json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON},
@@ -453,6 +473,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non
             target=model,
             dataset_glob_pattern="*.jsonl",
             prompt_template={"template": "Answer {{item.input}}"},
+            metric_bundler=None,
         )
 
         assert job is expected_job
@@ -463,6 +484,7 @@ def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> Non
             target=model,
             dataset_glob_pattern="*.jsonl",
             prompt_template={"template": "Answer {{item.input}}"},
+            metric_bundler=None,
         )
 
     def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None:
@@ -484,6 +506,7 @@ def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None:
             target=None,
             dataset_glob_pattern=None,
             prompt_template=None,
+            metric_bundler=None,
         )
 
 
@@ -564,10 +587,79 @@ def test_run_uses_local_executor_execution(self, mocker: MockerFixture) -> None:
         remote_evaluate.assert_not_called()
 
 
-def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None:
+def test_sync_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None:
+    platform = _SyncPlatform()
+    metric_bundler = mocker.Mock()
+    metric_bundler.bundle.side_effect = AssertionError("local execution should not bundle metrics")
+    executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform), metric_bundler=metric_bundler)
+    expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
+    sdk_evaluator = mocker.Mock()
+    sdk_evaluator.run_sync.return_value = expected
+    sdk_evaluator_cls = mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator)
+    metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
+    dataset = [{"expected": "a", "output": "a"}]
+
+    result = executor.evaluate(
+        metric=metric,
+        dataset=dataset,
+        params=RunConfig(parallelism=2),
+    )
+
+    assert result is expected
+    sdk_evaluator_cls.assert_called_once_with()
+    sdk_evaluator.run_sync.assert_called_once_with(
+        metrics=metric,
+        dataset=dataset,
+        config=RunConfig(parallelism=2),
+        target=None,
+        dataset_glob_pattern=None,
+        prompt_template=None,
+    )
+    metric_bundler.bundle.assert_not_called()
+
+
+def test_sync_executor_evaluate_resolves_fileset_ref_before_calling_sdk(mocker: MockerFixture) -> None:
     platform = _SyncPlatform()
     executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform))
     expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
+    sdk_evaluator = mocker.Mock()
+    sdk_evaluator.run_sync.return_value = expected
+    mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator)
+    downloaded_path = Path("/tmp/downloaded-dataset")
+    download_dataset_sync = mocker.patch(
+        "nemo_evaluator.sdk._executor.download_dataset_sync",
+        return_value=downloaded_path,
+    )
+    metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
+    dataset = FilesetRef(root="default/helpsteer2")
+
+    result = executor.evaluate(
+        metric=metric,
+        dataset=dataset,
+        dataset_glob_pattern="validation/*.jsonl",
+    )
+
+    assert result is expected
+    download_dataset_sync.assert_called_once()
+    assert download_dataset_sync.call_args.kwargs["sdk"] is platform
+    assert download_dataset_sync.call_args.kwargs["dataset"] == FilesetRef(root="default/helpsteer2#validation/*.jsonl")
+    sdk_evaluator.run_sync.assert_called_once_with(
+        metrics=metric,
+        dataset=downloaded_path,
+        config=RunConfig(),
+        target=None,
+        dataset_glob_pattern=None,
+        prompt_template=None,
+    )
+
+
+def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None:
+    platform = _SyncPlatform()
+    executor = _SyncEvaluatorPluginExecutor(
+        platform=cast(NeMoPlatform, platform),
+        metric_bundler=CloudpickleMetricBundler(),
+    )
+    expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
     job_resource = mocker.Mock(spec=EvaluatorJobResource)
     job_resource.get_result.return_value = expected
     create = mocker.patch.object(executor, "create", return_value=job_resource)
@@ -582,20 +674,12 @@ def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: Mocke
     )
 
     assert result == expected
-    create.assert_called_once_with(
-        spec=EvaluateSpec.model_validate(
-            {
-                "metric": {
-                    "type": "exact-match",
-                    "reference": "{{item.expected}}",
-                    "candidate": "{{item.output}}",
-                },
-                "dataset": [{"expected": "a", "output": "a"}],
-                "params": {"limit_samples": None, "parallelism": 2},
-            }
-        ),
-        workspace="platform-ws",
-    )
+    create.assert_called_once()
+    assert create.call_args.kwargs["workspace"] == "platform-ws"
+    created_spec = create.call_args.kwargs["spec"]
+    assert _single_metric(created_spec).metric_type == "exact-match"
+    assert created_spec.dataset == [{"expected": "a", "output": "a"}]
+    assert created_spec.params == RunConfig(parallelism=2)
     job_resource.wait_until_done.assert_called_once_with(
         poll_interval_seconds=10.0,
         job_timeout_seconds=3600.0,
@@ -662,7 +746,7 @@ async def test_async_executor_creates_evaluator_job(mocker: MockerFixture) -> No
     assert job.name == "job-123"
     assert job.job.status == PlatformJobStatus.CREATED
     assert job.job.spec is not None
-    assert _single_metric(job.job.spec).type == MetricType.EXACT_MATCH
+    assert _single_metric(job.job.spec).metric_type == "exact-match"
     platform._client.post.assert_awaited_once_with(
         "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs",
         json={"spec": _EXACT_MATCH_EVALUATE_SPEC_JSON},
@@ -794,6 +878,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture)
             target=model,
             dataset_glob_pattern="*.jsonl",
             prompt_template={"template": "Answer {{item.input}}"},
+            metric_bundler=None,
         )
 
         assert job is expected_job
@@ -804,6 +889,7 @@ async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture)
             target=model,
             dataset_glob_pattern="*.jsonl",
             prompt_template={"template": "Answer {{item.input}}"},
+            metric_bundler=None,
         )
 
     @pytest.mark.asyncio
@@ -826,6 +912,7 @@ async def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None:
             target=None,
             dataset_glob_pattern=None,
             prompt_template=None,
+            metric_bundler=None,
         )
 
 
@@ -934,10 +1021,48 @@ async def test_async_executor_remote_submit_uses_platform_async_client_headers_a
     http_client_cls.assert_not_called()
 
 
+@pytest.mark.asyncio
+async def test_async_executor_evaluate_calls_sdk_directly_without_bundling(mocker: MockerFixture) -> None:
+    platform = _AsyncPlatform()
+    metric_bundler = mocker.Mock()
+    metric_bundler.bundle.side_effect = AssertionError("local execution should not bundle metrics")
+    executor = _AsyncEvaluatorPluginExecutor(
+        platform=cast(AsyncNeMoPlatform, platform),
+        metric_bundler=metric_bundler,
+    )
+    expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
+    sdk_evaluator = mocker.Mock()
+    sdk_evaluator.run = AsyncMock(return_value=expected)
+    sdk_evaluator_cls = mocker.patch("nemo_evaluator.sdk._executor.SDKEvaluator", return_value=sdk_evaluator)
+    metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}")
+    dataset = [{"expected": "a", "output": "a"}]
+
+    result = await executor.evaluate(
+        metric=metric,
+        dataset=dataset,
+        params=RunConfig(parallelism=2),
+    )
+
+    assert result is expected
+    sdk_evaluator_cls.assert_called_once_with()
+    sdk_evaluator.run.assert_awaited_once_with(
+        metrics=metric,
+        dataset=dataset,
+        config=RunConfig(parallelism=2),
+        target=None,
+        dataset_glob_pattern=None,
+        prompt_template=None,
+    )
+    metric_bundler.bundle.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None:
     platform = _AsyncPlatform()
-    executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform))
+    executor = _AsyncEvaluatorPluginExecutor(
+        platform=cast(AsyncNeMoPlatform, platform),
+        metric_bundler=CloudpickleMetricBundler(),
+    )
     expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[]))
     job_resource = mocker.Mock(spec=AsyncEvaluatorJobResource)
     job_resource.wait_until_done = AsyncMock()
@@ -954,20 +1079,12 @@ async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker
     )
 
     assert result == expected
-    create.assert_awaited_once_with(
-        spec=EvaluateSpec.model_validate(
-            {
-                "metric": {
-                    "type": "exact-match",
-                    "reference": "{{item.expected}}",
-                    "candidate": "{{item.output}}",
-                },
-                "dataset": [{"expected": "a", "output": "a"}],
-                "params": {"limit_samples": None, "parallelism": 2},
-            }
-        ),
-        workspace="platform-ws",
-    )
+    create.assert_awaited_once()
+    assert create.call_args.kwargs["workspace"] == "platform-ws"
+    created_spec = create.call_args.kwargs["spec"]
+    assert _single_metric(created_spec).metric_type == "exact-match"
+    assert created_spec.dataset == [{"expected": "a", "output": "a"}]
+    assert created_spec.params == RunConfig(parallelism=2)
     job_resource.wait_until_done.assert_awaited_once_with(
         poll_interval_seconds=10.0,
         job_timeout_seconds=3600.0,
diff --git a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py
index a52bc5b8a4..0dc343c571 100644
--- a/plugins/nemo-evaluator/tests/test_sdk_job_resources.py
+++ b/plugins/nemo-evaluator/tests/test_sdk_job_resources.py
@@ -29,6 +29,8 @@
     metric_job_status_details_value,
     metric_job_status_value,
 )
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler
+from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
 from nemo_evaluator_sdk.values.results import (
     AggregatedMetricResult,
     AggregateRangeScore,
@@ -44,11 +46,9 @@
     "name": "job-123",
     "status": "created",
     "spec": {
-        "metric": {
-            "type": "exact-match",
-            "reference": "{{item.expected}}",
-            "candidate": "{{item.output}}",
-        },
+        "metric": CloudpickleMetricBundler()
+        .bundle(ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"))
+        .model_dump(mode="json"),
         "dataset": [{"expected": "a", "output": "a"}],
     },
 }
diff --git a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py
index da2c696e7b..e3ed33ce8d 100644
--- a/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py
+++ b/plugins/nemo-evaluator/tests/test_standalone_sdk_backend.py
@@ -107,7 +107,11 @@ def test_evaluate_remote_delegates_to_resource_executor_remote_path(self, mocker
         result = NMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request)
 
         assert result is expected
-        remote_evaluate.assert_called_once_with(metric=metric, request=request)
+        remote_evaluate.assert_called_once_with(
+            metric=metric,
+            request=request,
+            metric_bundler=None,
+        )
         local_evaluate.assert_not_called()
 
     def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: MockerFixture) -> None:
@@ -127,7 +131,10 @@ def test_evaluate_benchmark_local_delegates_to_resource_executor(self, mocker: M
         result = NMPBackend(resource).evaluate_benchmark(metrics=metrics, request=request)
 
         assert result is expected
-        evaluate_benchmark.assert_called_once_with(metrics=metrics, request=request)
+        evaluate_benchmark.assert_called_once_with(
+            metrics=metrics,
+            request=request,
+        )
 
     def test_evaluate_benchmark_remote_raises_without_local_run(self, mocker: MockerFixture) -> None:
         resource = Evaluator(cast(NeMoPlatform, _SyncPlatform()))
@@ -206,7 +213,11 @@ async def test_evaluate_remote_delegates_to_resource_executor_remote_path(self,
         result = await AsyncNMPBackend(resource, execution_mode="remote").evaluate(metric=metric, request=request)
 
         assert result is expected
-        remote_evaluate.assert_awaited_once_with(metric=metric, request=request)
+        remote_evaluate.assert_awaited_once_with(
+            metric=metric,
+            request=request,
+            metric_bundler=None,
+        )
         local_evaluate.assert_not_awaited()
 
     @pytest.mark.asyncio
@@ -227,7 +238,10 @@ async def test_evaluate_benchmark_local_delegates_to_resource_executor(self, moc
         result = await AsyncNMPBackend(resource).evaluate_benchmark(metrics=metrics, request=request)
 
         assert result is expected
-        evaluate_benchmark.assert_awaited_once_with(metrics=metrics, request=request)
+        evaluate_benchmark.assert_awaited_once_with(
+            metrics=metrics,
+            request=request,
+        )
 
     @pytest.mark.asyncio
     async def test_evaluate_benchmark_remote_raises_without_local_run(self, mocker: MockerFixture) -> None:
diff --git a/scripts/run_evaluator_local_bundle_path.py b/scripts/run_evaluator_local_bundle_path.py
new file mode 100644
index 0000000000..f4b42c891f
--- /dev/null
+++ b/scripts/run_evaluator_local_bundle_path.py
@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Smoke-test evaluator plugin local execution with runtime metrics."""
+
+from __future__ import annotations
+
+import httpx
+from nemo_evaluator.sdk.resources import Evaluator as PluginEvaluator
+from nemo_evaluator.sdk.standalone_sdk.backend import NMPBackend
+from nemo_evaluator_sdk import Evaluator as SDKEvaluator
+from nemo_evaluator_sdk.metrics.base import MetricBundle, MetricBundler
+from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_evaluator_sdk.values import RunConfig
+from nemo_platform import NeMoPlatform
+
+
+class CustomContainsMetric:
+    type = "custom-contains"
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [MetricOutputSpec.continuous_score("contains")]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        expected = str(input.row.data["expected"]).lower()
+        candidate = str(input.row.data["model_output"]).lower()
+        return MetricResult(outputs=[MetricOutput(name="contains", value=float(expected in candidate))])
+
+
+class _FailIfBundled(MetricBundler):
+    def bundle(self, metric: Metric) -> MetricBundle:
+        raise AssertionError("local evaluator execution should not bundle metrics")
+
+    def unbundle(self, metric: MetricBundle) -> Metric:
+        raise AssertionError("local evaluator execution should not unbundle metrics")
+
+
+def main() -> int:
+    with httpx.Client() as http_client:
+        platform = NeMoPlatform(
+            base_url="http://localhost:8000",
+            workspace="default",
+            http_client=http_client,
+        )
+        plugin_resource = PluginEvaluator(platform)
+        backend = NMPBackend(
+            plugin_resource,
+            execution_mode="local",
+            metric_bundler=_FailIfBundled(),
+        )
+        evaluator = SDKEvaluator(client=backend)
+        dataset = [
+            {"expected": "blue", "model_output": "Blue"},
+            {"expected": "Jupiter", "model_output": "Jupiter is the largest planet"},
+        ]
+
+        exact_result = evaluator.run_sync(
+            metrics=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}"),
+            dataset=dataset,
+            config=RunConfig(parallelism=2),
+        )
+        exact_score = exact_result.aggregate_scores.scores[0]
+        custom_result = evaluator.run_sync(
+            metrics=CustomContainsMetric(),
+            dataset=dataset,
+            config=RunConfig(parallelism=2),
+        )
+        custom_score = custom_result.aggregate_scores.scores[0]
+        multi_result = evaluator.run_sync(
+            metrics=[
+                ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}"),
+                CustomContainsMetric(),
+            ],
+            dataset=dataset,
+            config=RunConfig(parallelism=2),
+        )
+
+        print(f"exact_rows={len(exact_result.row_scores)}")
+        print(f"exact_score_name={exact_score.name}")
+        print(f"exact_mean={exact_score.mean}")
+        print(f"custom_rows={len(custom_result.row_scores)}")
+        print(f"custom_score_name={custom_score.name}")
+        print(f"custom_mean={custom_score.mean}")
+        print(f"multi_rows={len(multi_result.row_scores)}")
+        print(f"multi_scores={[score.name for score in multi_result.aggregate_scores.scores]}")
+        print(f"multi_means={[score.mean for score in multi_result.aggregate_scores.scores]}")
+
+        assert len(exact_result.row_scores) == 2
+        assert exact_score.name == "exact-match.exact-match"
+        assert exact_score.mean == 0.5
+        assert len(custom_result.row_scores) == 2
+        assert custom_score.name == "custom-contains.contains"
+        assert custom_score.mean == 1.0
+        assert len(multi_result.row_scores) == 2
+        assert [score.name for score in multi_result.aggregate_scores.scores] == [
+            "exact-match.exact-match",
+            "custom-contains.contains",
+        ]
+        assert [score.mean for score in multi_result.aggregate_scores.scores] == [0.5, 1.0]
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/scripts/run_evaluator_remote_bundle_path.py b/scripts/run_evaluator_remote_bundle_path.py
new file mode 100644
index 0000000000..de782a9983
--- /dev/null
+++ b/scripts/run_evaluator_remote_bundle_path.py
@@ -0,0 +1,701 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Smoke-test evaluator plugin remote execution with bundled metrics."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Sequence
+from contextlib import contextmanager
+from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from threading import Thread
+
+import httpx
+from nemo_evaluator.sdk._executor import _build_evaluate_spec
+from nemo_evaluator.sdk.resources import Evaluator as PluginEvaluator
+from nemo_evaluator.sdk.standalone_sdk.backend import NMPBackend
+from nemo_evaluator_sdk import Evaluator as SDKEvaluator
+from nemo_evaluator_sdk.enums import ModelFormat
+from nemo_evaluator_sdk.execution.config import EvaluationRequest
+from nemo_evaluator_sdk.metrics.bleu import BLEUMetric
+from nemo_evaluator_sdk.metrics.cloudpickle import CloudpickleMetricBundler
+from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+from nemo_evaluator_sdk.metrics.f1 import F1Metric
+from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
+from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric
+from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_evaluator_sdk.metrics.ragas import (
+    AgentGoalAccuracyMetric,
+    AnswerAccuracyMetric,
+    ContextEntityRecallMetric,
+    ContextPrecisionMetric,
+    ContextRecallMetric,
+    ContextRelevanceMetric,
+    FaithfulnessMetric,
+    NoiseSensitivityMetric,
+    ResponseGroundednessMetric,
+    ResponseRelevancyMetric,
+    ToolCallAccuracyMetric,
+    TopicAdherenceMetric,
+)
+from nemo_evaluator_sdk.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric
+from nemo_evaluator_sdk.metrics.rouge import ROUGEMetric
+from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric
+from nemo_evaluator_sdk.metrics.tool_calling import ToolCallingMetric
+from nemo_evaluator_sdk.metrics.utils import metric_type_name
+from nemo_evaluator_sdk.values import InferenceParams, Model, RunConfig, SecretRef
+from nemo_evaluator_sdk.values.scores import JSONScoreParser, RangeScore, RemoteScore
+from nemo_platform import NeMoPlatform
+from openai import AsyncOpenAI
+
+DatasetRow = dict[str, object]
+
+
+@dataclass(frozen=True)
+class MetricCase:
+    name: str
+    metric: Metric
+    expected_aggregate_scores: tuple[str, ...]
+
+
+class CustomContainsMetric:
+    type = "custom-contains"
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [MetricOutputSpec.continuous_score("contains")]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        expected = str(input.row.data["expected"]).lower()
+        candidate = str(input.row.data["model_output"]).lower()
+        return MetricResult(outputs=[MetricOutput(name="contains", value=float(expected in candidate))])
+
+
+async def _fake_judge_inference(
+    model: Model,
+    request: dict,
+    max_retries: int | None,
+    *,
+    client: AsyncOpenAI | None = None,
+    api_key: str | None = None,
+    default_headers: dict | None = None,
+    timeout: float | None = None,
+) -> dict:
+    del model, request, max_retries, client, api_key, default_headers, timeout
+    return {"choices": [{"message": {"content": json.dumps({"helpfulness": 4})}}]}
+
+
+class _RemoteMetricHandler(BaseHTTPRequestHandler):
+    def do_POST(self) -> None:
+        content_length = int(self.headers.get("content-length", "0"))
+        raw_body = self.rfile.read(content_length) if content_length else b"{}"
+        request = json.loads(raw_body.decode("utf-8"))
+        if request.get("evaluator_name") == "nat-quality":
+            response = {"result": {"score": 0.9}}
+        else:
+            response = {"result": {"quality": 0.75}}
+        body = json.dumps(response).encode("utf-8")
+        self.send_response(200)
+        self.send_header("content-type", "application/json")
+        self.send_header("content-length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, format: str, *args: object) -> None:
+        del format, args
+
+
+@contextmanager
+def _remote_metric_server():
+    server = ThreadingHTTPServer(("127.0.0.1", 0), _RemoteMetricHandler)
+    thread = Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield f"http://127.0.0.1:{server.server_port}"
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+
+def _base_url() -> str:
+    return os.environ.get("NEMO_PLATFORM_BASE_URL", "http://127.0.0.1:8080")
+
+
+def _workspace() -> str:
+    return os.environ.get("NEMO_PLATFORM_WORKSPACE", "default")
+
+
+def _poll_interval_seconds() -> float:
+    return float(os.environ.get("NEMO_EVALUATOR_SMOKE_POLL_INTERVAL_SECONDS", "2"))
+
+
+def _job_timeout_seconds() -> float:
+    return float(os.environ.get("NEMO_EVALUATOR_SMOKE_JOB_TIMEOUT_SECONDS", "600"))
+
+
+def _pending_timeout_seconds() -> float:
+    return float(os.environ.get("NEMO_EVALUATOR_SMOKE_PENDING_TIMEOUT_SECONDS", "120"))
+
+
+def _live_nvidia_enabled() -> bool:
+    return os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_NVIDIA") == "1"
+
+
+def _nvidia_secret_ref() -> SecretRef:
+    return SecretRef(root=os.environ.get("NEMO_EVALUATOR_SMOKE_NVIDIA_SECRET", "nvidia-api-key"))
+
+
+def _ensure_live_nvidia_secret(platform: NeMoPlatform) -> None:
+    api_key = os.environ.get("NVIDIA_API_KEY")
+    if not api_key:
+        raise RuntimeError("NEMO_EVALUATOR_SMOKE_LIVE_NVIDIA=1 requires NVIDIA_API_KEY in the script environment")
+
+    secret_name = _nvidia_secret_ref().root
+    try:
+        platform.secrets.retrieve(secret_name, workspace=_workspace())
+    except Exception:
+        platform.secrets.create(name=secret_name, value=api_key, workspace=_workspace())
+    else:
+        platform.secrets.update(secret_name, value=api_key, workspace=_workspace())
+
+
+def _nvidia_judge_model() -> Model:
+    return Model(
+        url=os.environ.get(
+            "NEMO_EVALUATOR_SMOKE_NVIDIA_CHAT_URL",
+            "https://integrate.api.nvidia.com/v1/chat/completions",
+        ),
+        name=os.environ.get("NEMO_EVALUATOR_SMOKE_NVIDIA_JUDGE_MODEL", "meta/llama-3.1-70b-instruct"),
+        api_key_secret=_nvidia_secret_ref(),
+        format=ModelFormat.NVIDIA_NIM,
+    )
+
+
+def _nvidia_embeddings_model() -> Model:
+    return Model(
+        url=os.environ.get(
+            "NEMO_EVALUATOR_SMOKE_NVIDIA_EMBEDDINGS_URL",
+            "https://integrate.api.nvidia.com/v1/embeddings",
+        ),
+        name=os.environ.get("NEMO_EVALUATOR_SMOKE_NVIDIA_EMBEDDINGS_MODEL", "nvidia/nv-embedqa-e5-v5"),
+        api_key_secret=_nvidia_secret_ref(),
+        format=ModelFormat.NVIDIA_NIM,
+    )
+
+
+def _live_llm_inference_params() -> InferenceParams:
+    return InferenceParams(
+        temperature=0.0,
+        max_tokens=int(os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_MAX_TOKENS", "512")),
+    )
+
+
+def _live_ragas_inference_params() -> InferenceParams:
+    return InferenceParams.model_validate(
+        {
+            "temperature": 0.0,
+            "max_tokens": int(os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_MAX_TOKENS", "512")),
+            "request_timeout": float(os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_REQUEST_TIMEOUT_SECONDS", "90")),
+            "max_retries": int(os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_MAX_RETRIES", "0")),
+            "max_workers": int(os.environ.get("NEMO_EVALUATOR_SMOKE_LIVE_MAX_WORKERS", "1")),
+        }
+    )
+
+
+def _dataset() -> list[DatasetRow]:
+    return [
+        {
+            "expected": "blue",
+            "model_output": "Blue",
+            "left_text": "prefix needle suffix",
+            "right_text": "needle",
+            "left_number": "42",
+            "right_number": "42",
+            "expected_tool_calls": [
+                {
+                    "id": "call_1",
+                    "type": "function",
+                    "function": {"name": "lookup_weather", "arguments": {"city": "Halifax"}},
+                }
+            ],
+            "response": {
+                "choices": [
+                    {
+                        "message": {
+                            "tool_calls": [
+                                {
+                                    "id": "call_1",
+                                    "type": "function",
+                                    "function": {
+                                        "name": "lookup_weather",
+                                        "arguments": '{"city": "Halifax"}',
+                                    },
+                                }
+                            ]
+                        }
+                    }
+                ]
+            },
+        },
+        {
+            "expected": "Jupiter",
+            "model_output": "Jupiter is the largest planet",
+            "left_text": "another haystack with needle",
+            "right_text": "needle",
+            "left_number": "7.5",
+            "right_number": "7.5",
+            "expected_tool_calls": [
+                {
+                    "id": "call_2",
+                    "type": "function",
+                    "function": {"name": "lookup_planet", "arguments": {"name": "Jupiter"}},
+                }
+            ],
+            "response": {
+                "choices": [
+                    {
+                        "message": {
+                            "tool_calls": [
+                                {
+                                    "id": "call_2",
+                                    "type": "function",
+                                    "function": {
+                                        "name": "lookup_planet",
+                                        "arguments": '{"name": "Jupiter"}',
+                                    },
+                                }
+                            ]
+                        }
+                    }
+                ]
+            },
+        },
+    ]
+
+
+def _judge_model() -> Model:
+    return Model(
+        url="https://judge.example.test/v1/chat/completions",
+        name="judge-model",
+        format=ModelFormat.OPEN_AI,
+    )
+
+
+def _embeddings_model() -> Model:
+    return Model(
+        url="https://judge.example.test/v1/embeddings",
+        name="embedding-model",
+        format=ModelFormat.OPEN_AI,
+    )
+
+
+def _llm_judge_metric() -> LLMJudgeMetric:
+    metric = LLMJudgeMetric(
+        model=_judge_model(),
+        scores=[
+            RangeScore(
+                name="helpfulness",
+                minimum=1,
+                maximum=5,
+                parser=JSONScoreParser(json_path="helpfulness"),
+            )
+        ],
+        prompt_template="Judge: {{item.expected}} -> {{item.model_output}}",
+    )
+    metric.set_inference_fn(_fake_judge_inference)
+    return metric
+
+
+def _live_llm_judge_metric() -> LLMJudgeMetric:
+    return LLMJudgeMetric(
+        model=_nvidia_judge_model(),
+        scores=[
+            RangeScore(
+                name="correctness",
+                minimum=1,
+                maximum=5,
+                parser=JSONScoreParser(json_path="correctness"),
+            )
+        ],
+        prompt_template={
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "Score the candidate against the expected answer. Return only JSON.",
+                },
+                {
+                    "role": "user",
+                    "content": "Expected: {{item.expected}}\nCandidate: {{item.model_output}}",
+                },
+            ]
+        },
+        inference=_live_llm_inference_params(),
+    )
+
+
+def _live_rag_dataset() -> list[DatasetRow]:
+    return [
+        {
+            "user_input": "What is the capital of France?",
+            "retrieved_contexts": [
+                "Paris is the capital and largest city of France.",
+                "Berlin is the capital of Germany.",
+            ],
+            "response": "The capital of France is Paris.",
+            "reference": "Paris is the capital of France.",
+        }
+    ]
+
+
+def _live_agentic_dataset() -> list[DatasetRow]:
+    return [
+        {
+            "user_input": [
+                {"content": "What's the weather in Paris?", "type": "human"},
+                {
+                    "content": "Let me check.",
+                    "type": "ai",
+                    "tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}],
+                },
+                {"content": "Sunny, 22C", "type": "tool"},
+                {"content": "It's sunny and 22C in Paris.", "type": "ai"},
+            ],
+            "reference": "The agent checked the weather for Paris and reported the result.",
+            "reference_tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}],
+            "reference_topics": ["weather", "Paris"],
+        }
+    ]
+
+
+def _live_rag_metric_cases() -> list[MetricCase]:
+    judge_model = _nvidia_judge_model()
+    inference = _live_ragas_inference_params()
+    return [
+        MetricCase("answer_accuracy", AnswerAccuracyMetric(judge_model=judge_model, inference=inference), ()),
+        MetricCase("context_relevance", ContextRelevanceMetric(judge_model=judge_model, inference=inference), ()),
+        MetricCase(
+            "response_groundedness",
+            ResponseGroundednessMetric(judge_model=judge_model, inference=inference),
+            (),
+        ),
+        MetricCase("context_recall", ContextRecallMetric(judge_model=judge_model, inference=inference), ()),
+        MetricCase("context_precision", ContextPrecisionMetric(judge_model=judge_model, inference=inference), ()),
+        MetricCase(
+            "context_entity_recall", ContextEntityRecallMetric(judge_model=judge_model, inference=inference), ()
+        ),
+        MetricCase(
+            "response_relevancy",
+            ResponseRelevancyMetric(
+                judge_model=judge_model,
+                embeddings_model=_nvidia_embeddings_model(),
+                inference=inference,
+                strictness=1,
+            ),
+            (),
+        ),
+        MetricCase("faithfulness", FaithfulnessMetric(judge_model=judge_model, inference=inference), ()),
+        MetricCase("noise_sensitivity", NoiseSensitivityMetric(judge_model=judge_model, inference=inference), ()),
+    ]
+
+
+def _live_agentic_metric_cases() -> list[MetricCase]:
+    judge_model = _nvidia_judge_model()
+    inference = _live_ragas_inference_params()
+    return [
+        MetricCase(
+            "topic_adherence",
+            TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model, inference=inference),
+            (),
+        ),
+        MetricCase("tool_call_accuracy", ToolCallAccuracyMetric(), ()),
+        MetricCase(
+            "agent_goal_accuracy",
+            AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model, inference=inference),
+            (),
+        ),
+    ]
+
+
+def _bundle_only_metric_cases() -> list[MetricCase]:
+    judge_model = _judge_model()
+    return [
+        MetricCase("topic_adherence", TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model), ()),
+        MetricCase("tool_call_accuracy", ToolCallAccuracyMetric(), ()),
+        MetricCase("agent_goal_accuracy", AgentGoalAccuracyMetric(judge_model=judge_model), ()),
+        MetricCase("answer_accuracy", AnswerAccuracyMetric(judge_model=judge_model), ()),
+        MetricCase("context_relevance", ContextRelevanceMetric(judge_model=judge_model), ()),
+        MetricCase("response_groundedness", ResponseGroundednessMetric(judge_model=judge_model), ()),
+        MetricCase("context_recall", ContextRecallMetric(judge_model=judge_model), ()),
+        MetricCase("context_precision", ContextPrecisionMetric(judge_model=judge_model), ()),
+        MetricCase("context_entity_recall", ContextEntityRecallMetric(judge_model=judge_model), ()),
+        MetricCase(
+            "response_relevancy",
+            ResponseRelevancyMetric(judge_model=judge_model, embeddings_model=_embeddings_model()),
+            (),
+        ),
+        MetricCase("faithfulness", FaithfulnessMetric(judge_model=judge_model), ()),
+        MetricCase("noise_sensitivity", NoiseSensitivityMetric(judge_model=judge_model), ()),
+    ]
+
+
+def _metric_cases(remote_metric_url: str) -> list[MetricCase]:
+    return [
+        MetricCase(
+            name="exact_match",
+            metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}"),
+            expected_aggregate_scores=("exact-match.exact-match",),
+        ),
+        MetricCase(
+            name="f1",
+            metric=F1Metric(reference="{{item.expected}}", candidate="{{item.model_output}}"),
+            expected_aggregate_scores=("f1.f1",),
+        ),
+        MetricCase(
+            name="bleu",
+            metric=BLEUMetric(references=["{{item.expected}}"], candidate="{{item.model_output}}"),
+            expected_aggregate_scores=("bleu.sentence", "bleu.corpus"),
+        ),
+        MetricCase(
+            name="rouge",
+            metric=ROUGEMetric(reference="{{item.expected}}", candidate="{{item.model_output}}"),
+            expected_aggregate_scores=(
+                "rouge.rouge_1_score",
+                "rouge.rouge_2_score",
+                "rouge.rouge_3_score",
+                "rouge.rouge_L_score",
+            ),
+        ),
+        MetricCase(
+            name="string_check",
+            metric=StringCheckMetric(
+                operation="contains",
+                left_template="{{item.left_text}}",
+                right_template="{{item.right_text}}",
+            ),
+            expected_aggregate_scores=("string-check.string-check",),
+        ),
+        MetricCase(
+            name="number_check",
+            metric=NumberCheckMetric(
+                operation="equals",
+                left_template="{{item.left_number}}",
+                right_template="{{item.right_number}}",
+            ),
+            expected_aggregate_scores=("number-check.number-check",),
+        ),
+        MetricCase(
+            name="tool_calling",
+            metric=ToolCallingMetric(reference="{{item.expected_tool_calls}}"),
+            expected_aggregate_scores=(
+                "tool-calling.function_name_accuracy",
+                "tool-calling.function_name_and_args_accuracy",
+            ),
+        ),
+        MetricCase(
+            name="custom_protocol",
+            metric=CustomContainsMetric(),
+            expected_aggregate_scores=("custom-contains.contains",),
+        ),
+        MetricCase(
+            name="llm_judge",
+            metric=_llm_judge_metric(),
+            expected_aggregate_scores=("llm-judge.helpfulness",),
+        ),
+        MetricCase(
+            name="remote",
+            metric=RemoteMetric(
+                url=remote_metric_url,
+                body={"prompt": "{{item.expected}}", "candidate": "{{item.model_output}}"},
+                scores=[RemoteScore(name="quality", parser=JSONScoreParser(json_path="$.result.quality"))],
+                max_retries=0,
+            ),
+            expected_aggregate_scores=("remote.quality",),
+        ),
+        MetricCase(
+            name="nemo_agent_toolkit_remote",
+            metric=NemoAgentToolkitRemoteMetric(
+                url=remote_metric_url,
+                evaluator_name="nat-quality",
+                max_retries=0,
+            ),
+            expected_aggregate_scores=("nemo-agent-toolkit-remote.nat-quality",),
+        ),
+    ]
+
+
+def _aggregate_score_names(result) -> list[str]:
+    return [score.name for score in result.aggregate_scores.scores]
+
+
+def _expected_names(metric_cases: Sequence[MetricCase]) -> list[str]:
+    return [
+        f"{metric_type_name(metric_case.metric)}.{output.name}"
+        for metric_case in metric_cases
+        for output in metric_case.metric.output_spec()
+    ]
+
+
+def _assert_bundle_round_trips(metric_cases: Sequence[MetricCase], bundler: CloudpickleMetricBundler) -> None:
+    for metric_case in metric_cases:
+        bundle = bundler.bundle(metric_case.metric)
+        hydrated = bundler.unbundle(bundle)
+        if [output.name for output in hydrated.output_spec()] != [
+            output.name for output in metric_case.metric.output_spec()
+        ]:
+            raise AssertionError(f"bundle round trip changed output spec for {metric_case.name}")
+
+
+def _assert_scores_present(actual: Sequence[str], expected: Sequence[str]) -> None:
+    missing = [score_name for score_name in expected if score_name not in actual]
+    if missing:
+        raise AssertionError(f"missing aggregate scores {missing}; actual scores: {list(actual)}")
+
+
+def _assert_no_metric_errors(result, label: str) -> None:
+    errors = [row_score.metric_errors for row_score in result.row_scores if row_score.metric_errors]
+    if errors:
+        raise AssertionError(f"{label} had row metric errors: {errors}")
+
+
+def _assert_no_all_nan_scores(result, label: str) -> None:
+    all_nan_scores = [
+        score.name for score in result.aggregate_scores.scores if score.count > 0 and score.nan_count == score.count
+    ]
+    if all_nan_scores:
+        raise AssertionError(f"{label} produced only NaN values for scores: {all_nan_scores}")
+
+
+def _run_live_metric_job(
+    *,
+    plugin_resource: PluginEvaluator,
+    bundler: CloudpickleMetricBundler,
+    label: str,
+    metric_cases: Sequence[MetricCase],
+    dataset: list[DatasetRow],
+):
+    spec = _build_evaluate_spec(
+        metrics=[metric_case.metric for metric_case in metric_cases],
+        request=EvaluationRequest(dataset=dataset, params=RunConfig(parallelism=1)),
+        metric_bundler=bundler,
+    )
+    job = plugin_resource._executor.create(spec=spec, workspace=_workspace())
+    job.wait_until_done(
+        poll_interval_seconds=_poll_interval_seconds(),
+        job_timeout_seconds=_job_timeout_seconds(),
+        pending_timeout_seconds=_pending_timeout_seconds(),
+    )
+    result = job.get_result()
+    score_names = _aggregate_score_names(result)
+    _assert_scores_present(score_names, _expected_names(metric_cases))
+    _assert_no_metric_errors(result, label)
+    _assert_no_all_nan_scores(result, label)
+    print(f"{label}_job={job.name}")
+    print(f"{label}_scores={score_names}")
+    print(f"{label}_means={[score.mean for score in result.aggregate_scores.scores]}")
+    return result
+
+
+def main() -> int:
+    dataset = _dataset()
+    config = RunConfig(parallelism=2)
+    bundler = CloudpickleMetricBundler()
+
+    with _remote_metric_server() as remote_metric_url, httpx.Client(timeout=httpx.Timeout(30.0)) as http_client:
+        metric_cases = _metric_cases(remote_metric_url)
+        bundle_only_metric_cases = _bundle_only_metric_cases()
+        _assert_bundle_round_trips([*metric_cases, *bundle_only_metric_cases], bundler)
+        platform = NeMoPlatform(
+            base_url=_base_url(),
+            workspace=_workspace(),
+            http_client=http_client,
+        )
+        plugin_resource = PluginEvaluator(platform)
+        backend = NMPBackend(
+            plugin_resource,
+            execution_mode="remote",
+            metric_bundler=bundler,
+        )
+        evaluator = SDKEvaluator(client=backend)
+        if _live_nvidia_enabled():
+            _ensure_live_nvidia_secret(platform)
+
+        exact_result = evaluator.run_sync(
+            metrics=metric_cases[0].metric,
+            dataset=dataset,
+            config=config,
+        )
+        exact_score = exact_result.aggregate_scores.scores[0]
+        custom_result = evaluator.run_sync(
+            metrics=CustomContainsMetric(),
+            dataset=dataset,
+            config=config,
+        )
+        custom_score = custom_result.aggregate_scores.scores[0]
+
+        multi_spec = _build_evaluate_spec(
+            metrics=[metric_case.metric for metric_case in metric_cases],
+            request=EvaluationRequest(dataset=dataset, params=config),
+            metric_bundler=bundler,
+        )
+        multi_job = plugin_resource._executor.create(spec=multi_spec, workspace=_workspace())
+        multi_job.wait_until_done(
+            poll_interval_seconds=_poll_interval_seconds(),
+            job_timeout_seconds=_job_timeout_seconds(),
+            pending_timeout_seconds=_pending_timeout_seconds(),
+        )
+        multi_result = multi_job.get_result()
+
+        print(f"exact_rows={len(exact_result.row_scores)}")
+        print(f"exact_score_name={exact_score.name}")
+        print(f"exact_mean={exact_score.mean}")
+        print(f"custom_rows={len(custom_result.row_scores)}")
+        print(f"custom_score_name={custom_score.name}")
+        print(f"custom_mean={custom_score.mean}")
+        print(f"multi_job={multi_job.name}")
+        print(f"multi_rows={len(multi_result.row_scores)}")
+        print(f"multi_scores={_aggregate_score_names(multi_result)}")
+        print(f"multi_means={[score.mean for score in multi_result.aggregate_scores.scores]}")
+        print(f"remote_job_metric_cases={[metric_case.name for metric_case in metric_cases]}")
+        print(f"bundle_only_metric_cases={[metric_case.name for metric_case in bundle_only_metric_cases]}")
+
+        assert len(exact_result.row_scores) == 2
+        assert exact_score.name == "exact-match.exact-match"
+        assert exact_score.mean == 0.5
+        assert len(custom_result.row_scores) == 2
+        assert custom_score.name == "custom-contains.contains"
+        assert custom_score.mean == 1.0
+        assert len(multi_result.row_scores) == 2
+        multi_score_names = _aggregate_score_names(multi_result)
+        for metric_case in metric_cases:
+            _assert_scores_present(multi_score_names, metric_case.expected_aggregate_scores)
+        if _live_nvidia_enabled():
+            live_llm_cases = [MetricCase("live_llm_judge", _live_llm_judge_metric(), ("llm-judge.correctness",))]
+            _run_live_metric_job(
+                plugin_resource=plugin_resource,
+                bundler=bundler,
+                label="live_llm_judge",
+                metric_cases=live_llm_cases,
+                dataset=dataset[:1],
+            )
+            _run_live_metric_job(
+                plugin_resource=plugin_resource,
+                bundler=bundler,
+                label="live_ragas_rag",
+                metric_cases=_live_rag_metric_cases(),
+                dataset=_live_rag_dataset(),
+            )
+            _run_live_metric_job(
+                plugin_resource=plugin_resource,
+                bundler=bundler,
+                label="live_ragas_agentic",
+                metric_cases=_live_agentic_metric_cases(),
+                dataset=_live_agentic_dataset(),
+            )
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml
index 21da423850..40e5b34472 100644
--- a/sdk/python/nemo-platform/pyproject.toml
+++ b/sdk/python/nemo-platform/pyproject.toml
@@ -61,6 +61,7 @@ evaluator = [
   "openai>=1.61.0",
   "sacrebleu>=2.5.1",
   "rouge_score==0.1.2",
+  "cloudpickle>=3.1.1",
 ]
 
 [project.entry-points."nemo.skills"]
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py
index 6d33101875..74a2e62a7b 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/__init__.py
@@ -12,13 +12,22 @@
     EvaluationError,
     EvaluationPhase,
 )
+from nemo_platform.beta.evaluator.metrics.base import (
+    MetricBundle,
+    MetricBundlePayload,
+    MetricBundler,
+    MetricBundlingError,
+    MetricMetadata,
+)
 from nemo_platform.beta.evaluator.metrics.bleu import BLEUMetric
+from nemo_platform.beta.evaluator.metrics.cloudpickle import CloudpickleMetricBundler, CloudpickleMetricPayload
 from nemo_platform.beta.evaluator.metrics.exact_match import ExactMatchMetric
 from nemo_platform.beta.evaluator.metrics.f1 import F1Metric
 from nemo_platform.beta.evaluator.metrics.llm_judge import LLMJudgeMetric
 from nemo_platform.beta.evaluator.metrics.number_check import NumberCheckMetric
 from nemo_platform.beta.evaluator.metrics.protocol import (
     Metric,
+    MetricTypeName,
     validate_metric_result,
 )
 from nemo_platform.beta.evaluator.metrics.remote import NemoAgentToolkitRemoteMetric, RemoteMetric
@@ -69,6 +78,10 @@
 __all__ = [
     "BLEUMetric",
     "Agent",
+    "MetricBundle",
+    "MetricBundlePayload",
+    "CloudpickleMetricBundler",
+    "CloudpickleMetricPayload",
     "EvaluationError",
     "EvaluationPhase",
     "DatasetLoadError",
@@ -85,11 +98,15 @@
     "InferenceStructuredOutput",
     "JSONScoreParser",
     "Metric",
+    "MetricBundler",
+    "MetricBundlingError",
     "MetricDescriptor",
     "MetricInput",
+    "MetricMetadata",
     "MetricOutput",
     "MetricOutputSpec",
     "MetricResult",
+    "MetricTypeName",
     "LLMJudgeMetric",
     "BooleanValue",
     "CandidateOutput",
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py
new file mode 100644
index 0000000000..b5d10b5442
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/base.py
@@ -0,0 +1,185 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Backend-neutral metric bundle models and protocols."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Mapping
+from typing import Any, Literal, Protocol, cast
+
+from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricOutputSpec, MetricTypeName, MetricWithSecrets
+from nemo_platform.beta.evaluator.values.common import SecretRef
+from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, field_serializer, field_validator, model_validator
+
+
+class MetricBundlingError(ValueError):
+    """Raised when a metric cannot be bundled or hydrated."""
+
+
+class MetricMetadata(BaseModel):
+    """User-facing metadata captured with a bundled metric."""
+
+    model_config = ConfigDict(extra="allow", revalidate_instances="never")
+
+    description: str | None = None
+    labels: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("labels")
+    @classmethod
+    def _labels_must_be_strings(cls, value: dict[str, str]) -> dict[str, str]:
+        for key, label_value in value.items():
+            if not isinstance(key, str) or not isinstance(label_value, str):
+                raise ValueError("metric labels must be a mapping of string keys to string values")
+        return value
+
+
+class BundledMetricOutputSpec(BaseModel):
+    """JSON-safe projection of a runtime metric output spec."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    description: str | None = None
+    value_json_schema: dict[str, Any]
+
+    @classmethod
+    def from_output_spec(cls, output: MetricOutputSpec) -> "BundledMetricOutputSpec":
+        """Capture the serializable contract for one runtime output."""
+        return cls(
+            name=output.name,
+            description=output.description,
+            value_json_schema=output.value_json_schema(),
+        )
+
+
+class MetricBundlePayload(BaseModel, ABC):
+    """Base class for concrete Pydantic metric bundle payloads."""
+
+    @property
+    @abstractmethod
+    def kind(self) -> str:
+        """Payload discriminator used to select the bundler implementation."""
+        ...
+
+
+_PAYLOAD_TYPES: dict[str, type[MetricBundlePayload]] = {}
+_BUNDLER_FACTORIES: dict[str, Callable[[], MetricBundler]] = {}
+
+
+def _payload_kind(payload: MetricBundlePayload) -> str:
+    kind = payload.kind
+    if not kind:
+        raise MetricBundlingError("metric bundle payload kind must not be empty")
+    return kind
+
+
+def register_metric_bundle_payload(kind: str, payload_type: type[MetricBundlePayload]) -> None:
+    """Register a concrete Pydantic payload model for a bundle kind."""
+    if not kind:
+        raise ValueError("metric bundle payload kind must not be empty")
+    _PAYLOAD_TYPES[kind] = payload_type
+
+
+def register_metric_bundler(kind: str, factory: Callable[[], MetricBundler]) -> None:
+    """Register a metric bundler factory for a payload kind."""
+    if not kind:
+        raise ValueError("metric bundle payload kind must not be empty")
+    _BUNDLER_FACTORIES[kind] = factory
+
+
+class MetricBundle(BaseModel):
+    """Standalone executable metric bundle entity used by backend execution."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    bundle_kind: Literal["metric-bundle"] = "metric-bundle"
+    bundle_format_version: Literal["v1"] = "v1"
+    metric_type: MetricTypeName
+    metadata: MetricMetadata = Field(default_factory=MetricMetadata)
+    outputs: list[BundledMetricOutputSpec] = Field(min_length=1)
+    secrets: dict[str, SecretRef] = Field(default_factory=dict)
+    payload: SerializeAsAny[MetricBundlePayload]
+    digest: str
+
+    @field_serializer("payload")
+    def _serialize_payload(self, payload: MetricBundlePayload) -> dict[str, Any]:
+        value = payload.model_dump(mode="json")
+        value["kind"] = _payload_kind(payload)
+        return value
+
+    @field_validator("payload", mode="before")
+    @classmethod
+    def _payload_must_have_kind(cls, value: object) -> object:
+        if isinstance(value, MetricBundlePayload):
+            return value
+        if not isinstance(value, Mapping):
+            raise ValueError("metric bundle payload must be an object")
+        payload_data = cast(Mapping[str, object], value)
+        kind = payload_data.get("kind")
+        if not isinstance(kind, str) or not kind:
+            raise ValueError("metric bundle payload must include a non-empty kind")
+        payload_type = _PAYLOAD_TYPES.get(kind)
+        if payload_type is None:
+            raise ValueError(f"unsupported metric bundle payload kind: {kind}")
+        return payload_type.model_validate(value)
+
+    @model_validator(mode="after")
+    def _output_names_must_be_unique(self) -> "MetricBundle":
+        names = [output.name for output in self.outputs]
+        duplicates = sorted({name for name in names if names.count(name) > 1})
+        if duplicates:
+            raise ValueError(f"duplicate metric output names: {duplicates}")
+        return self
+
+
+class MetricBundler(Protocol):
+    """Interface for metric bundle implementations."""
+
+    def bundle(self, metric: Metric) -> MetricBundle:
+        """Serialize an executable metric to a bundle entity."""
+        ...
+
+    def unbundle(self, metric: MetricBundle) -> Metric:
+        """Hydrate an executable metric from a bundle entity."""
+        ...
+
+
+def metric_bundler_for_payload(payload: MetricBundlePayload) -> MetricBundler:
+    """Create the bundler registered for a metric bundle payload."""
+    kind = _payload_kind(payload)
+    factory = _BUNDLER_FACTORIES.get(kind)
+    if factory is None:
+        raise MetricBundlingError(f"unsupported metric bundle payload kind: {kind}")
+    return factory()
+
+
+def validate_metric_type(metric: Metric) -> str:
+    """Return the runtime metric type after validating the protocol contract."""
+    value = metric.type
+    if not isinstance(value, str):
+        raise MetricBundlingError("metric type must be a string")
+    if not value:
+        raise MetricBundlingError("metric type must not be empty")
+    return value
+
+
+def metric_metadata(metric: Metric) -> MetricMetadata:
+    """Capture optional runtime metric metadata."""
+    description = getattr(metric, "description", None)
+    if description is not None and not isinstance(description, str):
+        raise MetricBundlingError("metric description must be a string when provided")
+
+    raw_labels = getattr(metric, "labels", None) or {}
+    if not isinstance(raw_labels, Mapping):
+        raise MetricBundlingError("metric labels must be a mapping when provided")
+    labels = dict(raw_labels)
+    return MetricMetadata(description=description, labels=labels)
+
+
+def metric_secrets(metric: Metric) -> dict[str, SecretRef]:
+    """Capture secret environment mappings needed to execute one metric."""
+    if not isinstance(metric, MetricWithSecrets):
+        return {}
+    return metric.secrets()
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/cloudpickle.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/cloudpickle.py
new file mode 100644
index 0000000000..98896587a1
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/cloudpickle.py
@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Cloudpickle-backed metric bundle implementation."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import pickle
+import platform
+from typing import Literal
+
+import cloudpickle
+from nemo_platform.beta.evaluator.metrics.base import (
+    BundledMetricOutputSpec,
+    MetricBundle,
+    MetricBundlePayload,
+    MetricBundler,
+    MetricBundlingError,
+    metric_metadata,
+    metric_secrets,
+    register_metric_bundle_payload,
+    register_metric_bundler,
+    validate_metric_type,
+)
+from nemo_platform.beta.evaluator.metrics.protocol import Metric
+from pydantic import ConfigDict, field_validator
+
+
+class CloudpickleMetricPayload(MetricBundlePayload):
+    """Cloudpickle payload for an executable metric object."""
+
+    model_config = ConfigDict(extra="ignore", ser_json_bytes="base64", val_json_bytes="base64")
+
+    python_version: str
+    cloudpickle_version: str
+    pickle_protocol: int
+    blob: bytes
+
+    @property
+    def kind(self) -> Literal["cloudpickle"]:
+        """Payload discriminator used by the metric bundle registry."""
+        return "cloudpickle"
+
+    @classmethod
+    def from_blob(cls, blob: bytes) -> CloudpickleMetricPayload:
+        """Create a JSON-safe cloudpickle payload from raw bytes."""
+        return cls(
+            python_version=platform.python_version(),
+            cloudpickle_version=cloudpickle.__version__,
+            pickle_protocol=pickle.HIGHEST_PROTOCOL,
+            blob=blob,
+        )
+
+    @field_validator("blob")
+    @classmethod
+    def _blob_must_not_be_empty(cls, value: bytes) -> bytes:
+        if not value:
+            raise ValueError("blob must not be empty")
+        try:
+            encoded = base64.b64encode(value).decode("ascii")
+            base64.b64decode(encoded.encode("ascii"), validate=True)
+        except Exception as e:
+            raise ValueError("blob must be JSON-serializable as base64") from e
+        return value
+
+    def blob_bytes(self) -> bytes:
+        """Return the decoded cloudpickle payload bytes."""
+        return self.blob
+
+
+class CloudpickleMetricBundler(MetricBundler):
+    """Cloudpickle-backed metric bundler.
+
+    Cloudpickle bundles execute arbitrary Python code when hydrated. This
+    implementation is intended for explicit opt-in development/MVP use.
+    """
+
+    def bundle(self, metric: Metric) -> MetricBundle:
+        """Serialize a runtime metric object to a cloudpickle bundle."""
+        if not isinstance(metric, Metric):
+            raise MetricBundlingError("object does not satisfy the Metric protocol")
+
+        blob = cloudpickle.dumps(metric, protocol=pickle.HIGHEST_PROTOCOL)
+        digest = hashlib.sha256(blob).hexdigest()
+        return MetricBundle(
+            metric_type=validate_metric_type(metric),
+            metadata=metric_metadata(metric),
+            outputs=[BundledMetricOutputSpec.from_output_spec(output) for output in metric.output_spec()],
+            secrets=metric_secrets(metric),
+            payload=CloudpickleMetricPayload.from_blob(blob),
+            digest=digest,
+        )
+
+    def unbundle(self, metric: MetricBundle) -> Metric:
+        """Hydrate a metric from a cloudpickle bundle."""
+        payload = CloudpickleMetricPayload.model_validate(metric.payload.model_dump(mode="python"))
+        blob = payload.blob_bytes()
+        digest = hashlib.sha256(blob).hexdigest()
+        if digest != metric.digest:
+            raise MetricBundlingError("metric bundle digest does not match payload")
+
+        hydrated_metric = cloudpickle.loads(blob)
+        if not isinstance(hydrated_metric, Metric):
+            raise MetricBundlingError("unbundled object does not satisfy the Metric protocol")
+
+        output_names = [output.name for output in hydrated_metric.output_spec()]
+        bundled_output_names = [output.name for output in metric.outputs]
+        if output_names != bundled_output_names:
+            raise MetricBundlingError("unbundled metric output spec does not match bundle metadata")
+        if validate_metric_type(hydrated_metric) != metric.metric_type:
+            raise MetricBundlingError("unbundled metric type does not match bundle metadata")
+        return hydrated_metric
+
+
+register_metric_bundle_payload("cloudpickle", CloudpickleMetricPayload)
+register_metric_bundler("cloudpickle", CloudpickleMetricBundler)
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py
index 21b00d74f5..918f0b3918 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/protocol.py
@@ -7,12 +7,13 @@
 
 import math
 from collections.abc import Awaitable, Callable
-from typing import Any, Protocol, runtime_checkable
+from typing import Annotated, Any, Protocol, runtime_checkable
 
 from nemo_platform.beta.evaluator.values.common import SecretRef
-from pydantic import BaseModel, ConfigDict, Field, RootModel, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, RootModel, StringConstraints, field_serializer, field_validator
 
 SecretResolver = Callable[[str], Awaitable[str | None]]
+MetricTypeName = Annotated[str, StringConstraints(min_length=1)]
 
 
 class DatasetRow(BaseModel):
@@ -126,16 +127,9 @@ class MetricDescriptor(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    type: str
+    type: MetricTypeName
     outputs: list[MetricOutputSpec] = Field(min_length=1)
 
-    @field_validator("type")
-    @classmethod
-    def _type_must_not_be_empty(cls, value: str) -> str:
-        if not value:
-            raise ValueError("metric type must not be empty")
-        return value
-
     @field_validator("outputs")
     @classmethod
     def _output_names_must_be_unique(cls, value: list[MetricOutputSpec]) -> list[MetricOutputSpec]:
@@ -174,7 +168,7 @@ class Metric(Protocol):
     """Shared row-scoring primitive for SDK runtime metrics."""
 
     @property
-    def type(self) -> str:
+    def type(self) -> MetricTypeName:
         """Return the public metric key/type identifier."""
         ...
 
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py
index 67685fc413..c4f3ceb35c 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/__init__.py
@@ -15,6 +15,7 @@
     MetricOutput,
     MetricOutputSpec,
     MetricResult,
+    MetricTypeName,
 )
 from nemo_platform.beta.evaluator.values.agents import Agent
 from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes
@@ -114,6 +115,7 @@
     "MetricOutput",
     "MetricOutputSpec",
     "MetricResult",
+    "MetricTypeName",
     "MetricScore",
     "Model",
     "DatasetInput",
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py
index 408c5621aa..aab93d709c 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/metrics.py
@@ -20,6 +20,7 @@
 from nemo_platform.beta.evaluator.dataset_schemas.compatibility import merge_metric_required_schemas
 from nemo_platform.beta.evaluator.dataset_schemas.templates import infer_required_schema_from_template
 from nemo_platform.beta.evaluator.enums import MetricType
+from nemo_platform.beta.evaluator.metrics.protocol import MetricTypeName
 from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes
 from nemo_platform.beta.evaluator.values.dataset_schemas import InputSchema
 from nemo_platform.beta.evaluator.values.models import Model, ReasoningParams
@@ -129,7 +130,7 @@ class MetricBase(BaseModel):
 
     __entity_type__: ClassVar[str] = "metric"
 
-    type: str = Field(description="The type of metric. Used as a discriminator for the metric type.")
+    type: MetricTypeName = Field(description="The type of metric. Used as a discriminator for the metric type.")
     description: str | None = Field(default=None, description="Human-readable description of the metric.")
     labels: dict[str, str] = Field(
         default_factory=dict, description="Labels are key-value pairs that can be used for grouping and filtering."
diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py
index 9e9639efad..a74b558af9 100644
--- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py
+++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py
@@ -217,10 +217,10 @@ def _fetch_secret(*, api_base_url: str, principal: Principal | None, workspace:
     except URLError as exc:
         raise RuntimeError(f"failed to fetch secret {workspace}/{secret_name}: {exc.reason}") from exc
 
-    data = payload.get("data")
-    if not isinstance(data, str):
-        raise RuntimeError(f"failed to fetch secret {workspace}/{secret_name}: missing string data field")
-    return data
+    value = payload.get("value")
+    if not isinstance(value, str):
+        raise RuntimeError(f"failed to fetch secret {workspace}/{secret_name}: missing string value field")
+    return value
 
 
 def _validate_http_url(url: str, field_name: str) -> None:
diff --git a/services/core/jobs/tests/controllers/test_subprocess_runtime.py b/services/core/jobs/tests/controllers/test_subprocess_runtime.py
index 39780b45fd..2b6a6b7ff6 100644
--- a/services/core/jobs/tests/controllers/test_subprocess_runtime.py
+++ b/services/core/jobs/tests/controllers/test_subprocess_runtime.py
@@ -46,7 +46,7 @@ def test_inject_secret_env_vars():
     }
 
     response = MagicMock()
-    response.read.return_value = json.dumps({"data": "secret-value"}).encode("utf-8")
+    response.read.return_value = json.dumps({"value": "secret-value"}).encode("utf-8")
     response.__enter__.return_value = response
     response.__exit__.return_value = None
 
@@ -60,6 +60,21 @@ def test_inject_secret_env_vars():
     assert request.get_header("X-nmp-principal-on-behalf-of") == "creator@example.com"
 
 
+def test_inject_secret_env_vars_rejects_missing_value_field():
+    env = {
+        NEMO_JOB_SECRETS_ENVVAR: "HF_TOKEN=default/hf-token",
+        "NMP_SECRETS_URL": "http://secrets.example",
+    }
+    response = MagicMock()
+    response.read.return_value = json.dumps({"data": "secret-value"}).encode("utf-8")
+    response.__enter__.return_value = response
+    response.__exit__.return_value = None
+
+    with patch("nmp.core.jobs.controllers.backends.subprocess_runtime.urlopen", return_value=response):
+        with pytest.raises(RuntimeError, match="missing string value field"):
+            inject_secret_env_vars(env.copy())
+
+
 def test_inject_secret_env_vars_rejects_non_http_secrets_url():
     env = {
         NEMO_JOB_SECRETS_ENVVAR: "HF_TOKEN=default/hf-token",
diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl
index 8405ae9fab..75d18f0790 100644
--- a/third_party/licenses.jsonl
+++ b/third_party/licenses.jsonl
@@ -44,6 +44,7 @@
 {"name": "click-plugins", "license": "BSD-3-CLAUSE", "compatible": true}
 {"name": "click-repl", "license": "MIT", "compatible": true}
 {"name": "clickhouse-connect", "license": "APACHE-2.0", "compatible": true}
+{"name": "cloudpickle", "license": "BSD-3-CLAUSE", "compatible": true}
 {"name": "colorama", "license": "BSD-3-CLAUSE", "compatible": true}
 {"name": "colorlog", "license": "MIT", "compatible": true}
 {"name": "cryptography", "license": "APACHE-2.0", "compatible": true}
diff --git a/third_party/osv-licenses.json b/third_party/osv-licenses.json
index 7a72e08173..d6545b8d0b 100644
--- a/third_party/osv-licenses.json
+++ b/third_party/osv-licenses.json
@@ -465,6 +465,16 @@
             "Apache-2.0"
           ]
         },
+        {
+          "package": {
+            "name": "cloudpickle",
+            "version": "3.1.2",
+            "ecosystem": "PyPI"
+          },
+          "licenses": [
+            "BSD-3-Clause"
+          ]
+        },
         {
           "package": {
             "name": "colorama",
@@ -1916,7 +1926,7 @@
         {
           "package": {
             "name": "nvidia-nat-atif",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -1926,7 +1936,7 @@
         {
           "package": {
             "name": "nvidia-nat-config-optimizer",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -1936,7 +1946,7 @@
         {
           "package": {
             "name": "nvidia-nat-core",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -1946,7 +1956,7 @@
         {
           "package": {
             "name": "nvidia-nat-eval",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -1956,7 +1966,7 @@
         {
           "package": {
             "name": "nvidia-nat-langchain",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -1966,7 +1976,7 @@
         {
           "package": {
             "name": "nvidia-nat-opentelemetry",
-            "version": "1.7.0rc3",
+            "version": "1.7.0",
             "ecosystem": "PyPI"
           },
           "licenses": [
@@ -3242,7 +3252,7 @@
         {
           "package": {
             "name": "sqlfluff",
-            "version": "3.4.0",
+            "version": "4.1.0",
             "ecosystem": "PyPI"
           },
           "vulnerabilities": [
@@ -3461,29 +3471,67 @@
                 "nvd_published_at": null,
                 "severity": "HIGH"
               }
-            },
+            }
+          ],
+          "groups": [
             {
-              "modified": "2026-05-19T20:15:16Z",
-              "published": "2026-05-19T20:10:17Z",
-              "schema_version": "1.7.5",
-              "id": "GHSA-wmhf-fqc8-vxhh",
+              "ids": [
+                "GHSA-73jc-5mrq-prw7"
+              ],
               "aliases": [
-                "CVE-2026-46373"
+                "CVE-2026-46374",
+                "GHSA-73jc-5mrq-prw7"
               ],
-              "summary": "SQLFluff: Recursive Stack Overflow in Parser",
-              "details": "### Impact\n\nIn deployments where untrusted users can provide SQL queries to be linted, an untrusted user can submit a malicious query with deliberate excessive nesting to any application using the parser to trigger a Denial of Service through resource exhaustion.\n\n### Patches\n\nVersions 4.1.0 and up contain a configurable recursion limit, which is enabled by default, to prevent this manner of exploit.\n\n### Credit\n\nOri Nakar from Imperva Threat Research Team.",
-              "severity": [
-                {
-                  "type": "CVSS_V3",
-                  "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
-                }
+              "max_severity": "7.5"
+            }
+          ],
+          "licenses": [
+            "non-standard"
+          ]
+        },
+        {
+          "package": {
+            "name": "sqlmodel",
+            "version": "0.0.37",
+            "ecosystem": "PyPI"
+          },
+          "licenses": [
+            "MIT"
+          ]
+        },
+        {
+          "package": {
+            "name": "sse-starlette",
+            "version": "3.3.4",
+            "ecosystem": "PyPI"
+          },
+          "licenses": [
+            "BSD-3-Clause"
+          ]
+        },
+        {
+          "package": {
+            "name": "starlette",
+            "version": "0.52.1",
+            "ecosystem": "PyPI"
+          },
+          "vulnerabilities": [
+            {
+              "modified": "2026-05-22T13:30:15Z",
+              "published": "2026-05-22T13:11:38Z",
+              "schema_version": "1.7.5",
+              "id": "PYSEC-2026-161",
+              "aliases": [
+                "GHSA-86qp-5c8j-p5mr"
               ],
+              "summary": "Missing Host header validation poisons request.url.path, bypassing path-based security checks",
+              "details": "Starlette reconstructs the requested URL based on the HTTP Host request header and requested path, but does not perform any validation of the Host header value. This allows attackers to inject paths into the host part, prepending the actual path. However, routing in Starlette is based on the actual request path. This inconsistent interpretation of HTTP requests may lead to issues such as authentication bypass when the authentication depends on the reconstructed URL\u2019s path.",
               "affected": [
                 {
                   "package": {
                     "ecosystem": "PyPI",
-                    "name": "sqlfluff",
-                    "purl": "pkg:pypi/sqlfluff"
+                    "name": "starlette",
+                    "purl": "pkg:pypi/starlette"
                   },
                   "ranges": [
                     {
@@ -3493,243 +3541,241 @@
                           "introduced": "0"
                         },
                         {
-                          "fixed": "4.1.0"
+                          "fixed": "1.0.1"
                         }
                       ]
                     }
                   ],
                   "versions": [
-                    "0.0.1",
-                    "0.0.2",
-                    "0.0.3",
-                    "0.0.4",
-                    "0.0.5",
-                    "0.0.6",
-                    "0.0.7",
-                    "0.0.9",
                     "0.1.0",
                     "0.1.1",
+                    "0.1.10",
+                    "0.1.11",
+                    "0.1.12",
+                    "0.1.13",
+                    "0.1.14",
+                    "0.1.15",
+                    "0.1.16",
+                    "0.1.17",
                     "0.1.2",
                     "0.1.3",
                     "0.1.4",
                     "0.1.5",
+                    "0.1.6",
+                    "0.1.7",
+                    "0.1.8",
+                    "0.1.9",
                     "0.10.0",
                     "0.10.1",
+                    "0.10.2",
+                    "0.10.3",
+                    "0.10.4",
+                    "0.10.5",
+                    "0.10.6",
+                    "0.10.7",
                     "0.11.0",
                     "0.11.1",
                     "0.11.2",
+                    "0.11.3",
+                    "0.11.4",
                     "0.12.0",
+                    "0.12.0b1",
+                    "0.12.0b2",
+                    "0.12.0b3",
+                    "0.12.1",
+                    "0.12.10",
+                    "0.12.11",
+                    "0.12.12",
+                    "0.12.13",
+                    "0.12.2",
+                    "0.12.3",
+                    "0.12.4",
+                    "0.12.5",
+                    "0.12.6",
+                    "0.12.7",
+                    "0.12.8",
+                    "0.12.9",
                     "0.13.0",
                     "0.13.1",
                     "0.13.2",
+                    "0.13.3",
+                    "0.13.4",
+                    "0.13.5",
+                    "0.13.6",
+                    "0.13.7",
+                    "0.13.8",
+                    "0.14.0",
+                    "0.14.1",
+                    "0.14.2",
+                    "0.15.0",
+                    "0.16.0",
+                    "0.17.0",
+                    "0.17.1",
+                    "0.18.0",
+                    "0.19.0",
+                    "0.19.1",
                     "0.2.0",
                     "0.2.1",
                     "0.2.2",
                     "0.2.3",
-                    "0.2.4",
+                    "0.20.0",
+                    "0.20.1",
+                    "0.20.2",
+                    "0.20.3",
+                    "0.20.4",
+                    "0.21.0",
+                    "0.22.0",
+                    "0.23.0",
+                    "0.23.1",
+                    "0.24.0",
+                    "0.25.0",
+                    "0.26.0",
+                    "0.26.0.post1",
+                    "0.26.1",
+                    "0.27.0",
+                    "0.28.0",
+                    "0.29.0",
                     "0.3.0",
                     "0.3.1",
                     "0.3.2",
-                    "0.3.2.post1",
-                    "0.3.2.post2",
                     "0.3.3",
                     "0.3.4",
                     "0.3.5",
                     "0.3.6",
+                    "0.3.7",
+                    "0.30.0",
+                    "0.31.0",
+                    "0.31.1",
+                    "0.32.0",
+                    "0.32.0.post1",
+                    "0.33.0",
+                    "0.34.0",
+                    "0.35.0",
+                    "0.35.1",
+                    "0.36.0",
+                    "0.36.1",
+                    "0.36.2",
+                    "0.36.3",
+                    "0.37.0",
+                    "0.37.1",
+                    "0.37.2",
+                    "0.38.0",
+                    "0.38.1",
+                    "0.38.2",
+                    "0.38.3",
+                    "0.38.4",
+                    "0.38.5",
+                    "0.38.6",
+                    "0.39.0",
+                    "0.39.1",
+                    "0.39.2",
                     "0.4.0",
-                    "0.4.0a1",
-                    "0.4.0a2",
-                    "0.4.0a3",
                     "0.4.1",
+                    "0.4.2",
+                    "0.40.0",
+                    "0.41.0",
+                    "0.41.1",
+                    "0.41.2",
+                    "0.41.3",
+                    "0.42.0",
+                    "0.43.0",
+                    "0.44.0",
+                    "0.45.0",
+                    "0.45.1",
+                    "0.45.2",
+                    "0.45.3",
+                    "0.46.0",
+                    "0.46.1",
+                    "0.46.2",
+                    "0.47.0",
+                    "0.47.1",
+                    "0.47.2",
+                    "0.47.3",
+                    "0.48.0",
+                    "0.49.0",
+                    "0.49.1",
+                    "0.49.2",
+                    "0.49.3",
                     "0.5.0",
                     "0.5.1",
                     "0.5.2",
                     "0.5.3",
                     "0.5.4",
                     "0.5.5",
-                    "0.5.6",
+                    "0.50.0",
+                    "0.51.0",
+                    "0.52.0",
+                    "0.52.1",
                     "0.6.0",
-                    "0.6.0a1",
-                    "0.6.0a2",
                     "0.6.1",
                     "0.6.2",
                     "0.6.3",
-                    "0.6.4",
-                    "0.6.5",
-                    "0.6.6",
-                    "0.6.7",
-                    "0.6.8",
-                    "0.6.9",
                     "0.7.0",
-                    "0.7.0a1",
-                    "0.7.0a2",
-                    "0.7.0a3",
-                    "0.7.0a5",
-                    "0.7.0a8",
                     "0.7.1",
+                    "0.7.2",
+                    "0.7.3",
+                    "0.7.4",
                     "0.8.0",
                     "0.8.1",
                     "0.8.2",
+                    "0.8.3",
+                    "0.8.4",
+                    "0.8.5",
+                    "0.8.6",
+                    "0.8.7",
+                    "0.8.8",
                     "0.9.0",
                     "0.9.1",
+                    "0.9.10",
+                    "0.9.11",
                     "0.9.2",
                     "0.9.3",
                     "0.9.4",
+                    "0.9.5",
+                    "0.9.6",
+                    "0.9.7",
+                    "0.9.8",
+                    "0.9.9",
                     "1.0.0",
-                    "1.1.0",
-                    "1.2.0",
-                    "1.2.1",
-                    "1.3.0",
-                    "1.3.1",
-                    "1.3.2",
-                    "1.4.0",
-                    "1.4.1",
-                    "1.4.2",
-                    "1.4.3",
-                    "1.4.4",
-                    "1.4.5",
-                    "2.0.0",
-                    "2.0.0a1",
-                    "2.0.0a2",
-                    "2.0.0a3",
-                    "2.0.0a4",
-                    "2.0.0a5",
-                    "2.0.0a6",
-                    "2.0.1",
-                    "2.0.2",
-                    "2.0.3",
-                    "2.0.4",
-                    "2.0.5",
-                    "2.0.6",
-                    "2.0.7",
-                    "2.1.0",
-                    "2.1.1",
-                    "2.1.2",
-                    "2.1.3",
-                    "2.1.4",
-                    "2.2.0",
-                    "2.2.1",
-                    "2.3.0",
-                    "2.3.1",
-                    "2.3.2",
-                    "2.3.3",
-                    "2.3.4",
-                    "2.3.5",
-                    "3.0.0",
-                    "3.0.0a1",
-                    "3.0.0a2",
-                    "3.0.0a3",
-                    "3.0.0a4",
-                    "3.0.0a5",
-                    "3.0.0a6",
-                    "3.0.1",
-                    "3.0.2",
-                    "3.0.3",
-                    "3.0.4",
-                    "3.0.5",
-                    "3.0.6",
-                    "3.0.7",
-                    "3.1.0",
-                    "3.1.1",
-                    "3.2.0",
-                    "3.2.1",
-                    "3.2.2",
-                    "3.2.3",
-                    "3.2.4",
-                    "3.2.5",
-                    "3.3.0",
-                    "3.3.1",
-                    "3.4.0",
-                    "3.4.1",
-                    "3.4.2",
-                    "3.5.0",
-                    "4.0.0",
-                    "4.0.0a1",
-                    "4.0.0a2",
-                    "4.0.0a3",
-                    "4.0.1.post1",
-                    "4.0.3",
-                    "4.0.4",
-                    "4.0.4a1"
+                    "1.0.0rc1"
                   ],
                   "database_specific": {
-                    "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/05/GHSA-wmhf-fqc8-vxhh/GHSA-wmhf-fqc8-vxhh.json"
+                    "source": "https://github.com/pypa/advisory-database/blob/main/vulns/starlette/PYSEC-2026-161.yaml"
                   }
                 }
               ],
               "references": [
                 {
-                  "type": "WEB",
-                  "url": "https://github.com/sqlfluff/sqlfluff/security/advisories/GHSA-wmhf-fqc8-vxhh"
+                  "type": "EVIDENCE",
+                  "url": "https://github.com/Kludex/starlette/security/advisories/GHSA-86qp-5c8j-p5mr"
                 },
                 {
-                  "type": "PACKAGE",
-                  "url": "https://github.com/sqlfluff/sqlfluff"
+                  "type": "ADVISORY",
+                  "url": "https://github.com/Kludex/starlette/security/advisories/GHSA-86qp-5c8j-p5mr"
+                },
+                {
+                  "type": "ADVISORY",
+                  "url": "https://www.x41-dsec.de/lab/advisories/x41-2026-002-starlette/"
+                },
+                {
+                  "type": "FIX",
+                  "url": "https://github.com/Kludex/starlette/commit/764dab0dcfb9033d75442d7a359645c9f94648c6"
                 }
-              ],
-              "database_specific": {
-                "cwe_ids": [
-                  "CWE-674"
-                ],
-                "github_reviewed": true,
-                "github_reviewed_at": "2026-05-19T20:10:17Z",
-                "nvd_published_at": null,
-                "severity": "HIGH"
-              }
+              ]
             }
           ],
           "groups": [
             {
               "ids": [
-                "GHSA-73jc-5mrq-prw7"
-              ],
-              "aliases": [
-                "CVE-2026-46374",
-                "GHSA-73jc-5mrq-prw7"
-              ],
-              "max_severity": "7.5"
-            },
-            {
-              "ids": [
-                "GHSA-wmhf-fqc8-vxhh"
+                "PYSEC-2026-161"
               ],
               "aliases": [
-                "CVE-2026-46373",
-                "GHSA-wmhf-fqc8-vxhh"
+                "GHSA-86qp-5c8j-p5mr",
+                "PYSEC-2026-161"
               ],
-              "max_severity": "7.5"
+              "max_severity": ""
             }
           ],
-          "licenses": [
-            "non-standard"
-          ]
-        },
-        {
-          "package": {
-            "name": "sqlmodel",
-            "version": "0.0.37",
-            "ecosystem": "PyPI"
-          },
-          "licenses": [
-            "MIT"
-          ]
-        },
-        {
-          "package": {
-            "name": "sse-starlette",
-            "version": "3.3.4",
-            "ecosystem": "PyPI"
-          },
-          "licenses": [
-            "BSD-3-Clause"
-          ]
-        },
-        {
-          "package": {
-            "name": "starlette",
-            "version": "0.52.1",
-            "ecosystem": "PyPI"
-          },
           "licenses": [
             "BSD-3-Clause"
           ]
@@ -4352,7 +4398,7 @@
     },
     {
       "name": "BSD-3-Clause",
-      "count": 33
+      "count": 34
     },
     {
       "name": "ISC",
diff --git a/third_party/requirements-main.txt b/third_party/requirements-main.txt
index 2cf07c01ad..987681a960 100644
--- a/third_party/requirements-main.txt
+++ b/third_party/requirements-main.txt
@@ -407,6 +407,7 @@ boto3==1.40.61 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (
     # via
     #   aiobotocore
     #   langchain-aws
+    #   nemo-agents-plugin
     #   ngcsdk
 botocore==1.40.61 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
     --hash=sha256:17ebae412692fd4824f99cde0f08d50126dc97954008e5ba2b522eb049238aa7 \
@@ -414,6 +415,7 @@ botocore==1.40.61 ; (platform_machine == 'arm64' and sys_platform == 'darwin') o
     # via
     #   aiobotocore
     #   boto3
+    #   nemo-agents-plugin
     #   ngcsdk
     #   s3transfer
 botocore-stubs==1.42.41 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
@@ -619,6 +621,10 @@ clickhouse-connect==0.15.1 ; (platform_machine == 'arm64' and sys_platform == 'd
     --hash=sha256:f2aaf5fc0bb3098c24f0d8ca7e4ecbe605a26957481dfca2c8cef9d1fad7b7ca \
     --hash=sha256:fa01fdb92db6bf72cb9509eecd0a0057a4558a4f40c02eebffbc2d61b644620e
     # via nmp-intake
+cloudpickle==3.1.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414 \
+    --hash=sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a
+    # via nemo-evaluator-sdk
 colorama==0.4.6 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
@@ -1524,7 +1530,9 @@ langchain==1.2.14 ; (platform_machine == 'arm64' and sys_platform == 'darwin') o
 langchain-aws==1.1.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
     --hash=sha256:1e2f8570328eae4907c3cf7e900dc68d8034ddc865d9dc96823c9f9d8cccb901 \
     --hash=sha256:8ec074615b42839e035354063717374c32c63f5028ef5221ba073fd5f3ef5e37
-    # via nvidia-nat-langchain
+    # via
+    #   nemo-agents-plugin
+    #   nvidia-nat-langchain
 langchain-classic==1.0.7 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
     --hash=sha256:d9d9be38f7aa534ed0259c2410432e34a1f80b1d491e686749bb55af56479be3 \
     --hash=sha256:debbec8065e69b95108d2652e8d5c44f4516e19aa8d716c02ed2211c3aee099d
@@ -2108,34 +2116,34 @@ nvidia-ml-py==13.595.45 ; (platform_machine == 'arm64' and sys_platform == 'darw
     #   nemo-platform-ext
     #   nemo-platform-sdk
     #   nmp-common
-nvidia-nat-atif==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:b95979e24aae0ba07277ec20e993ae910278b2bd0b00ce3415037f02694a1ebb
+nvidia-nat-atif==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:65d471a366dfafe75cf94428bdba3007bd3e1368487e2e16fd62484298635334
     # via
     #   nemo-agents-plugin
     #   nvidia-nat-core
     #   nvidia-nat-eval
-nvidia-nat-config-optimizer==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:a0e04114fc85df980b96766dec366bf4c7a2e08e6aaa4ce1531dd1bc6dadfcde
+nvidia-nat-config-optimizer==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:5b14cb63d1076f314f63039a4f659c0d4129f20b2bedc38b026ceee53da9e8f1
     # via nemo-agents-plugin
-nvidia-nat-core==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:590ae4bf36f79068605806de2044429f29001a206d31a7297a4f1dacbedf9eb1
+nvidia-nat-core==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:fb4691ad3437e0e8b8d84256d36da18085ba1fc6dd47861e7a7e64c6a808390c
     # via
     #   nemo-agents-example-calculator
     #   nemo-agents-plugin
     #   nvidia-nat-config-optimizer
     #   nvidia-nat-langchain
     #   nvidia-nat-opentelemetry
-nvidia-nat-eval==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:44adcc9f8534b3a0abfde521aabc1e450ea52ed3d57381fcaf9ace7cf67b15c4
+nvidia-nat-eval==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:724fa6410a7a66050b525d75660d7fb753ed83aba32c37d31111ae322d54a548
     # via
     #   nemo-agents-plugin
     #   nvidia-nat-config-optimizer
     #   nvidia-nat-langchain
-nvidia-nat-langchain==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:544d52d0e6d6ae8c6b9da11ee5b471ba119e1704b98ed1e3119ab18c88c0d87c
+nvidia-nat-langchain==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:e00812f9d2c602bb59cc006081a5deef2e71cc46357e70f51503780196f9cc20
     # via nemo-agents-plugin
-nvidia-nat-opentelemetry==1.7.0rc3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:dbe77666b0c8143291995a76a591441c243d41a3db5c1b79560d79a0885fed2c
+nvidia-nat-opentelemetry==1.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:e468fb4bd2a9e2fa4fb0d67daa1dea22b900c70301249d2785aa96f305808f7f
     # via
     #   nemo-agents-plugin
     #   nvidia-nat-langchain
@@ -3627,9 +3635,9 @@ sqlalchemy==2.0.48 ; (platform_machine == 'arm64' and sys_platform == 'darwin')
     #   nmp-intake
     #   optuna
     #   sqlmodel
-sqlfluff==3.4.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
-    --hash=sha256:115e3f1bf1dc1318c58426ba3299eb682642cb67b5d12d9ea7c42b5e23aeabd6 \
-    --hash=sha256:6e1ea2d39b20cc791a1a009c234afaf043b448c7f2eb1c11551316fb41f36f47
+sqlfluff==4.1.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
+    --hash=sha256:83dd4c081afb48c0af861833015a18b13d52726bfe52a286246dbd7a64b7d111 \
+    --hash=sha256:ae11123ca4a697abadbd2783f85f04e58c36e7dd26ae8024f400efccc6a44631
     # via data-designer-engine
 sqlmodel==0.0.37 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \
     --hash=sha256:2137a4045ef3fd66a917a7717ada959a1ceb3630d95e1f6aaab39dd2c0aef278 \
diff --git a/uv.lock b/uv.lock
index c6d36a74f7..34c80b4ee5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4954,6 +4954,7 @@ name = "nemo-evaluator-sdk"
 version = "0.0.0"
 source = { editable = "packages/nemo_evaluator_sdk" }
 dependencies = [
+    { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
@@ -4978,6 +4979,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "cloudpickle", specifier = ">=3.1.1" },
     { name = "jinja2", specifier = ">=3.1.6" },
     { name = "jsonpath-ng", specifier = ">=1.7.0" },
     { name = "jsonschema", specifier = ">=4.23.0" },
@@ -5146,6 +5148,7 @@ entities-service = [
     { name = "uvicorn", extra = ["standard"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
 ]
 evaluator = [
+    { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
@@ -5643,6 +5646,7 @@ requires-dist = [
     { name = "celery", marker = "extra == 'services'", specifier = ">=5.5.3" },
     { name = "clickhouse-connect", marker = "extra == 'intake-service'", specifier = ">=0.7,<1.0" },
     { name = "clickhouse-connect", marker = "extra == 'services'", specifier = ">=0.7,<1.0" },
+    { name = "cloudpickle", marker = "extra == 'evaluator'", specifier = ">=3.1.1" },
     { name = "data-designer", marker = "extra == 'data-designer-nemo'", specifier = "==0.6.0" },
     { name = "data-designer", marker = "extra == 'nemo-anonymizer-plugin'", specifier = "==0.6.0" },
     { name = "data-designer", marker = "extra == 'nemo-data-designer-plugin'", specifier = "==0.6.0" },
@@ -6245,6 +6249,7 @@ aiohttp = [
     { name = "httpx-aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
 ]
 evaluator = [
+    { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonpath-ng", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
     { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform != 'darwin' and sys_platform != 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'darwin' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'darwin' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-12-nemoplatform-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128') or (sys_platform == 'linux' and extra == 'extra-12-nemoplatform-cu128' and extra == 'extra-20-nmp-safe-synthesizer-cpu') or (sys_platform == 'linux' and extra == 'extra-20-nmp-safe-synthesizer-cpu' and extra == 'extra-20-nmp-safe-synthesizer-cu128')" },
@@ -6281,6 +6286,7 @@ pydantic-v2 = [
 requires-dist = [
     { name = "aiohttp", marker = "extra == 'aiohttp'" },
     { name = "anyio", specifier = ">=4.0.0,<5" },
+    { name = "cloudpickle", marker = "extra == 'evaluator'", specifier = ">=3.1.1" },
     { name = "distro", specifier = ">=1.7.0,<2" },
     { name = "docker", specifier = ">=7.0.0" },
     { name = "fsspec", specifier = ">=2023.1.0" },