chore(evaluator): detangle plugin from legacy service

packages/nemo_evaluator_sdk/examples/examples.py

@@ -6,16 +6,11 @@
 from __future__ import annotations
 
 import asyncio
-import gzip
 import json
 import logging
 import os
-import shutil
-import urllib.error
-import urllib.request
 from collections.abc import Sequence
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any
 
 from nemo_evaluator_sdk.execution.evaluator import Evaluator
 from nemo_evaluator_sdk.execution.values import EvaluationError
@@ -30,13 +25,11 @@
     Model,
     RangeScore,
     RunConfig,
-    RunConfigOnlineModel,
     SecretRef,
 )
 
 if TYPE_CHECKING:
     import numpy as np
-    from nemo_platform import AsyncNeMoPlatform
 
 
 # --- 1. Defining reusable metric configs and custom metrics ---
@@ -46,11 +39,8 @@
     "You are an evaluator. Rate the response's helpfulness from 0-4. "
     'Return only a JSON object with this shape: {"helpfulness": <integer>}.'
 )
-# Local evaluator and local plugin execution resolve this as an environment variable name.
+# Local evaluator execution resolves this as an environment variable name.
 DEFAULT_API_KEY_SECRET = os.getenv("NMP_EVALUATOR_DEFAULT_API_KEY_SECRET", "NVIDIA_API_KEY")
-DEFAULT_WORKSPACE = os.getenv("NMP_EVALUATOR_DEFAULT_WORKSPACE", "default")
-HELPSTEER2_VALIDATION_JSONL_URL = "https://huggingface.co/datasets/nvidia/HelpSteer2/resolve/main/validation.jsonl.gz"
-_EXAMPLES_DIR = Path(__file__).resolve().parent
 
 
 def configure_example_logging() -> None:
@@ -119,45 +109,6 @@ def configure_example_logging() -> None:
     },
 ]
 
-OFFLINE_HELPFULNESS_DATASET = [
-    {
-        "prompt": "What is the capital of France?",
-        "response": "Paris is the capital of France.",
-        "helpfulness": 4,
-    },
-    {
-        "prompt": "How do I make scrambled eggs?",
-        "response": "Eggs.",
-        "helpfulness": 1,
-    },
-]
-
-
-def get_helpsteer2_dataset() -> Path:
-    """Return a local HelpSteer2 validation JSONL path, downloading it when absent."""
-
-    dataset_dir = _EXAMPLES_DIR / "temp" / "helpsteer2-eval"
-    validation_jsonl = dataset_dir / "validation.jsonl"
-    if validation_jsonl.exists():
-        return validation_jsonl
-
-    dataset_dir.mkdir(parents=True, exist_ok=True)
-    validation_jsonl_gz = dataset_dir / "validation.jsonl.gz"
-    try:
-        with (
-            urllib.request.urlopen(HELPSTEER2_VALIDATION_JSONL_URL, timeout=60) as response,
-            validation_jsonl_gz.open("wb") as gz_file,
-        ):
-            shutil.copyfileobj(response, gz_file)
-    except urllib.error.URLError as e:
-        raise RuntimeError(
-            f"Failed to download HelpSteer2 validation dataset from {HELPSTEER2_VALIDATION_JSONL_URL}: {e}"
-        ) from e
-    with gzip.open(validation_jsonl_gz, "rb") as compressed_file, validation_jsonl.open("wb") as output_file:
-        shutil.copyfileobj(compressed_file, output_file)  # pyright: ignore[reportArgumentType]
-    return validation_jsonl
-
-
 ONLINE_CHAT_PROMPT_TEMPLATE = {"messages": [{"role": "user", "content": "{{item.prompt}}"}]}
 
 
@@ -171,44 +122,6 @@ def get_helpsteer2_dataset() -> Path:
 model_with_custom_headers = model.with_default_headers({"X-My-Header": "value"})
 
 
-async def ensure_remote_evaluator_api_key_secret(workspace: str, client: AsyncNeMoPlatform) -> str:
-    """Resolve API key secret name from env and ensure the secret exists on the platform."""
-    from nemo_platform import ConflictError, NotFoundError
-
-    # API service expects lowercase secret name; if we keep it uppercase, the request will fail.
-    secret_name = DEFAULT_API_KEY_SECRET.lower()
-    try:
-        await client.secrets.retrieve(secret_name, workspace=workspace)
-    except NotFoundError:
-        api_key = os.getenv(DEFAULT_API_KEY_SECRET) or os.getenv("NVIDIA_API_KEY") or os.getenv("NVIDIA_BUILD_API_KEY")
-        if api_key is None:
-            raise RuntimeError(
-                f"Remote online evaluation needs a platform secret named '{secret_name}' in workspace "
-                f"'{workspace}'. Set NVIDIA_BUILD_API_KEY or NVIDIA_API_KEY to let this "
-                "example create it, or create it manually with: "
-                f"nemo secrets create {secret_name} --data '<api-key>' --workspace {workspace}"
-            ) from None
-        try:
-            await client.secrets.create(workspace=workspace, name=secret_name, value=api_key)
-            print(f"Secret {workspace}/{secret_name} created")
-        except ConflictError:
-            pass
-    return secret_name
-
-
-async def model_with_valid_secret(
-    *,
-    execution_mode: Literal["local", "remote"],
-    workspace: str,
-    client: AsyncNeMoPlatform,
-) -> Model:
-    """Return a model configured for local or remote NeMo Platform example execution."""
-    if execution_mode == "remote":
-        secret_name = await ensure_remote_evaluator_api_key_secret(workspace, client)
-        return model.model_copy(update={"api_key_secret": SecretRef(root=secret_name)})
-    return model
-
-
 def create_helpfulness_metric(judge_model: Model) -> LLMJudgeMetric:
     """Build a reusable LLM judge metric for helpfulness scoring."""
     return LLMJudgeMetric(
@@ -626,121 +539,6 @@ def run_sync_example() -> None:
     result.print_summary()
 
 
-# --- 3. NeMo Platform evaluator plugin workflows ---
-async def run_nmp_online_metric_example() -> None:
-    """Run one online metric job locally through the evaluator plugin resource."""
-
-    _print_example_separator(run_nmp_online_metric_example.__name__)
-
-    from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend
-    from nemo_platform import AsyncNeMoPlatform
-
-    client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0)
-    try:
-        evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator))
-        result = await evaluator.run(
-            metrics=ExactMatchMetric(reference="{{item.reference}}"),
-            target=model,
-            dataset=ONLINE_EXACT_MATCH_DATASET,
-            prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
-            config=RunConfigOnlineModel(parallelism=4),
-        )
-    finally:
-        await client.close()
-
-    print("\nCompleted NeMo Platform online evaluator plugin job locally...")
-    result.print_summary()
-
-
-async def run_nmp_llm_judge_example(
-    is_online: bool = False,
-    limit_samples: int = 2,
-    execution_mode: Literal["local", "remote"] = "local",
-) -> None:
-    """Run a helpfulness judge job locally through the evaluator plugin resource."""
-
-    _print_example_separator(
-        run_nmp_llm_judge_example.__name__,
-        is_online=is_online,
-        limit_samples=limit_samples,
-        execution_mode=execution_mode,
-    )
-
-    from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend
-    from nemo_platform import AsyncNeMoPlatform
-
-    nemo_client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0)
-    evaluator_plugin_client = nemo_client.evaluator
-    try:
-        run_kwargs: dict[str, Any] = {}
-        model = await model_with_valid_secret(
-            execution_mode=execution_mode,
-            workspace=DEFAULT_WORKSPACE,
-            client=nemo_client,
-        )
-        evaluator = Evaluator(client=AsyncNMPBackend(evaluator_plugin_client, execution_mode=execution_mode))
-        params: RunConfig | RunConfigOnlineModel = RunConfig(limit_samples=limit_samples)
-
-        if is_online:
-            params = RunConfigOnlineModel(parallelism=4, limit_samples=limit_samples)
-            run_kwargs["target"] = model
-            run_kwargs["prompt_template"] = ONLINE_CHAT_PROMPT_TEMPLATE
-
-        result = await evaluator.run(
-            metrics=create_helpfulness_metric(model),
-            dataset=get_helpsteer2_dataset(),
-            config=params,
-            **run_kwargs,
-        )
-        result.print_summary()
-    finally:
-        await nemo_client.close()
-
-    judge_scores, human_scores = extract_helpfulness_scores(
-        result.row_scores,
-        judge_response_index=1 if is_online else 0,
-    )
-    print(f"\nEvaluated: {len(judge_scores)} samples")
-    if len(judge_scores):
-        print(f"judge avg: {judge_scores.mean()}")
-        print(f"human avg: {human_scores.mean()}")
-
-
-async def run_nmp_benchmark_example() -> None:
-    """Run one NeMo Platform online benchmark example with multiple metrics locally."""
-
-    _print_example_separator(run_nmp_benchmark_example.__name__)
-
-    from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend
-    from nemo_platform import AsyncNeMoPlatform
-
-    client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0)
-    try:
-        evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator))
-        exact_match = ExactMatchMetric(reference="{{item.reference}}")
-        contains_required_phrase = StringCheckMetric(
-            operation="contains",
-            left_template="{{sample.output_text}}",
-            right_template="{{item.required_phrase}}",
-        )
-
-        print("\nRunning local NeMo Platform online benchmark evaluation...")
-
-        benchmark_result = await evaluator.run(
-            metrics=[exact_match, contains_required_phrase],
-            target=model,
-            dataset=ONLINE_BENCHMARK_DATASET,
-            prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE,
-            config=RunConfigOnlineModel(parallelism=4),
-        )
-    finally:
-        await client.close()
-    benchmark_result.print_summary()
-    print(f"Online benchmark metric keys: {list(benchmark_result.per_metric)}")
-    print(f"Exact match scores: {benchmark_result.metric_result('exact-match').aggregate_scores.scores}")
-    print(f"String check scores: {benchmark_result.metric_result('string-check').aggregate_scores.scores}")
-
-
 async def run_examples() -> None:
     """Execute the example workflows exposed by this module.
 
@@ -757,12 +555,6 @@ async def run_examples() -> None:
     await run_local_benchmark_with_metric_failure_example()
     await run_local_metric_with_template_failure_example()
 
-    ##### NeMo Platform backend examples #####
-    ### !!! `uv sync --group enabled-plugins` before `nemo run services` to enable evaluator plugin !!!
-    await run_nmp_online_metric_example()
-    await run_nmp_llm_judge_example(is_online=True, execution_mode="remote")
-    await run_nmp_benchmark_example()
-
 
 if __name__ == "__main__":
     configure_example_logging()

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py

@@ -3,8 +3,6 @@
 
 """Dataset loader module for evaluator SDK runtime."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/datasets/loader.py
-
 import gzip
 import io
 import json

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md

@@ -29,19 +29,9 @@ result = await evaluator.run(
 )
 ```
 
-```python
-# Evaluator plugin execution through the plugin-owned SDK.
-from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend
-
-client = AsyncNeMoPlatform(workspace="default")
-evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator))
-result = await evaluator.run(metrics=metric, dataset=data)
-```
-
 ## Design Notes
 
 - `Evaluator()` uses `LocalBackend`.
-- `Evaluator(client=...)` accepts an evaluator backend object. For NeMo Platform
-  execution, wrap the mounted evaluator resource in `NMPBackend` or
-  `AsyncNMPBackend` from the evaluator plugin package.
-- `nemo_platform` and `nemo_evaluator` are optional for local SDK evaluation.
+- `Evaluator(client=...)` accepts an evaluator backend object. Platform-specific
+  backend adapters are provided by
+  [`nemo-evaluator-plugin`](../../../../../plugins/nemo-evaluator/README.md).

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py

@@ -3,8 +3,6 @@
 
 """Aggregation data structures and computations for metric results."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py
-
 import math
 from collections import OrderedDict, defaultdict
 from collections.abc import Mapping, Sequence

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/templates.py

@@ -3,8 +3,6 @@
 
 """Template rendering helpers for evaluator SDK runtime."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/templates.py
-
 import json
 from typing import Any
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/common.py

@@ -3,8 +3,6 @@
 
 """Common value types used throughout evaluator SDK runtime."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/values/common.py
-
 from enum import Enum
 
 from pydantic import Field, RootModel

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/datasets.py

@@ -3,8 +3,6 @@
 
 """Dataset-related value types for evaluator SDK runtime."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/values/datasets.py
-
 from typing import Any, TypeAlias
 
 import pyarrow as pa

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py

@@ -3,9 +3,6 @@
 
 """Result types for evaluator SDK runtime."""
 
-# Migrated from: services/evaluator/src/nmp/evaluator/app/values/results.py
-# Migrated from: services/evaluator/src/nmp/evaluator/app/values/scores.py
-
 from __future__ import annotations
 
 import json