diff --git a/packages/nemo_evaluator_sdk/examples/examples.py b/packages/nemo_evaluator_sdk/examples/examples.py index 6c8a187c3f..90c8a7a8a5 100644 --- a/packages/nemo_evaluator_sdk/examples/examples.py +++ b/packages/nemo_evaluator_sdk/examples/examples.py @@ -6,16 +6,11 @@ from __future__ import annotations import asyncio -import gzip import json import logging import os -import shutil -import urllib.error -import urllib.request from collections.abc import Sequence -from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any from nemo_evaluator_sdk.execution.evaluator import Evaluator from nemo_evaluator_sdk.execution.values import EvaluationError @@ -30,13 +25,11 @@ Model, RangeScore, RunConfig, - RunConfigOnlineModel, SecretRef, ) if TYPE_CHECKING: import numpy as np - from nemo_platform import AsyncNeMoPlatform # --- 1. Defining reusable metric configs and custom metrics --- @@ -46,11 +39,8 @@ "You are an evaluator. Rate the response's helpfulness from 0-4. " 'Return only a JSON object with this shape: {"helpfulness": }.' ) -# Local evaluator and local plugin execution resolve this as an environment variable name. +# Local evaluator execution resolves this as an environment variable name. DEFAULT_API_KEY_SECRET = os.getenv("NMP_EVALUATOR_DEFAULT_API_KEY_SECRET", "NVIDIA_API_KEY") -DEFAULT_WORKSPACE = os.getenv("NMP_EVALUATOR_DEFAULT_WORKSPACE", "default") -HELPSTEER2_VALIDATION_JSONL_URL = "https://huggingface.co/datasets/nvidia/HelpSteer2/resolve/main/validation.jsonl.gz" -_EXAMPLES_DIR = Path(__file__).resolve().parent def configure_example_logging() -> None: @@ -119,45 +109,6 @@ def configure_example_logging() -> None: }, ] -OFFLINE_HELPFULNESS_DATASET = [ - { - "prompt": "What is the capital of France?", - "response": "Paris is the capital of France.", - "helpfulness": 4, - }, - { - "prompt": "How do I make scrambled eggs?", - "response": "Eggs.", - "helpfulness": 1, - }, -] - - -def get_helpsteer2_dataset() -> Path: - """Return a local HelpSteer2 validation JSONL path, downloading it when absent.""" - - dataset_dir = _EXAMPLES_DIR / "temp" / "helpsteer2-eval" - validation_jsonl = dataset_dir / "validation.jsonl" - if validation_jsonl.exists(): - return validation_jsonl - - dataset_dir.mkdir(parents=True, exist_ok=True) - validation_jsonl_gz = dataset_dir / "validation.jsonl.gz" - try: - with ( - urllib.request.urlopen(HELPSTEER2_VALIDATION_JSONL_URL, timeout=60) as response, - validation_jsonl_gz.open("wb") as gz_file, - ): - shutil.copyfileobj(response, gz_file) - except urllib.error.URLError as e: - raise RuntimeError( - f"Failed to download HelpSteer2 validation dataset from {HELPSTEER2_VALIDATION_JSONL_URL}: {e}" - ) from e - with gzip.open(validation_jsonl_gz, "rb") as compressed_file, validation_jsonl.open("wb") as output_file: - shutil.copyfileobj(compressed_file, output_file) # pyright: ignore[reportArgumentType] - return validation_jsonl - - ONLINE_CHAT_PROMPT_TEMPLATE = {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} @@ -171,44 +122,6 @@ def get_helpsteer2_dataset() -> Path: model_with_custom_headers = model.with_default_headers({"X-My-Header": "value"}) -async def ensure_remote_evaluator_api_key_secret(workspace: str, client: AsyncNeMoPlatform) -> str: - """Resolve API key secret name from env and ensure the secret exists on the platform.""" - from nemo_platform import ConflictError, NotFoundError - - # API service expects lowercase secret name; if we keep it uppercase, the request will fail. - secret_name = DEFAULT_API_KEY_SECRET.lower() - try: - await client.secrets.retrieve(secret_name, workspace=workspace) - except NotFoundError: - api_key = os.getenv(DEFAULT_API_KEY_SECRET) or os.getenv("NVIDIA_API_KEY") or os.getenv("NVIDIA_BUILD_API_KEY") - if api_key is None: - raise RuntimeError( - f"Remote online evaluation needs a platform secret named '{secret_name}' in workspace " - f"'{workspace}'. Set NVIDIA_BUILD_API_KEY or NVIDIA_API_KEY to let this " - "example create it, or create it manually with: " - f"nemo secrets create {secret_name} --data '' --workspace {workspace}" - ) from None - try: - await client.secrets.create(workspace=workspace, name=secret_name, value=api_key) - print(f"Secret {workspace}/{secret_name} created") - except ConflictError: - pass - return secret_name - - -async def model_with_valid_secret( - *, - execution_mode: Literal["local", "remote"], - workspace: str, - client: AsyncNeMoPlatform, -) -> Model: - """Return a model configured for local or remote NeMo Platform example execution.""" - if execution_mode == "remote": - secret_name = await ensure_remote_evaluator_api_key_secret(workspace, client) - return model.model_copy(update={"api_key_secret": SecretRef(root=secret_name)}) - return model - - def create_helpfulness_metric(judge_model: Model) -> LLMJudgeMetric: """Build a reusable LLM judge metric for helpfulness scoring.""" return LLMJudgeMetric( @@ -626,121 +539,6 @@ def run_sync_example() -> None: result.print_summary() -# --- 3. NeMo Platform evaluator plugin workflows --- -async def run_nmp_online_metric_example() -> None: - """Run one online metric job locally through the evaluator plugin resource.""" - - _print_example_separator(run_nmp_online_metric_example.__name__) - - from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend - from nemo_platform import AsyncNeMoPlatform - - client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0) - try: - evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator)) - result = await evaluator.run( - metrics=ExactMatchMetric(reference="{{item.reference}}"), - target=model, - dataset=ONLINE_EXACT_MATCH_DATASET, - prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE, - config=RunConfigOnlineModel(parallelism=4), - ) - finally: - await client.close() - - print("\nCompleted NeMo Platform online evaluator plugin job locally...") - result.print_summary() - - -async def run_nmp_llm_judge_example( - is_online: bool = False, - limit_samples: int = 2, - execution_mode: Literal["local", "remote"] = "local", -) -> None: - """Run a helpfulness judge job locally through the evaluator plugin resource.""" - - _print_example_separator( - run_nmp_llm_judge_example.__name__, - is_online=is_online, - limit_samples=limit_samples, - execution_mode=execution_mode, - ) - - from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend - from nemo_platform import AsyncNeMoPlatform - - nemo_client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0) - evaluator_plugin_client = nemo_client.evaluator - try: - run_kwargs: dict[str, Any] = {} - model = await model_with_valid_secret( - execution_mode=execution_mode, - workspace=DEFAULT_WORKSPACE, - client=nemo_client, - ) - evaluator = Evaluator(client=AsyncNMPBackend(evaluator_plugin_client, execution_mode=execution_mode)) - params: RunConfig | RunConfigOnlineModel = RunConfig(limit_samples=limit_samples) - - if is_online: - params = RunConfigOnlineModel(parallelism=4, limit_samples=limit_samples) - run_kwargs["target"] = model - run_kwargs["prompt_template"] = ONLINE_CHAT_PROMPT_TEMPLATE - - result = await evaluator.run( - metrics=create_helpfulness_metric(model), - dataset=get_helpsteer2_dataset(), - config=params, - **run_kwargs, - ) - result.print_summary() - finally: - await nemo_client.close() - - judge_scores, human_scores = extract_helpfulness_scores( - result.row_scores, - judge_response_index=1 if is_online else 0, - ) - print(f"\nEvaluated: {len(judge_scores)} samples") - if len(judge_scores): - print(f"judge avg: {judge_scores.mean()}") - print(f"human avg: {human_scores.mean()}") - - -async def run_nmp_benchmark_example() -> None: - """Run one NeMo Platform online benchmark example with multiple metrics locally.""" - - _print_example_separator(run_nmp_benchmark_example.__name__) - - from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend - from nemo_platform import AsyncNeMoPlatform - - client = AsyncNeMoPlatform(workspace=DEFAULT_WORKSPACE, timeout=30000.0) - try: - evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator)) - exact_match = ExactMatchMetric(reference="{{item.reference}}") - contains_required_phrase = StringCheckMetric( - operation="contains", - left_template="{{sample.output_text}}", - right_template="{{item.required_phrase}}", - ) - - print("\nRunning local NeMo Platform online benchmark evaluation...") - - benchmark_result = await evaluator.run( - metrics=[exact_match, contains_required_phrase], - target=model, - dataset=ONLINE_BENCHMARK_DATASET, - prompt_template=ONLINE_CHAT_PROMPT_TEMPLATE, - config=RunConfigOnlineModel(parallelism=4), - ) - finally: - await client.close() - benchmark_result.print_summary() - print(f"Online benchmark metric keys: {list(benchmark_result.per_metric)}") - print(f"Exact match scores: {benchmark_result.metric_result('exact-match').aggregate_scores.scores}") - print(f"String check scores: {benchmark_result.metric_result('string-check').aggregate_scores.scores}") - - async def run_examples() -> None: """Execute the example workflows exposed by this module. @@ -757,12 +555,6 @@ async def run_examples() -> None: await run_local_benchmark_with_metric_failure_example() await run_local_metric_with_template_failure_example() - ##### NeMo Platform backend examples ##### - ### !!! `uv sync --group enabled-plugins` before `nemo run services` to enable evaluator plugin !!! - await run_nmp_online_metric_example() - await run_nmp_llm_judge_example(is_online=True, execution_mode="remote") - await run_nmp_benchmark_example() - if __name__ == "__main__": configure_example_logging() diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py index 0f7235aa8b..8a6bee0083 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py @@ -3,8 +3,6 @@ """Dataset loader module for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/datasets/loader.py - import gzip import io import json diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md index 08d03db451..274fbc0585 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/README.md @@ -29,19 +29,9 @@ result = await evaluator.run( ) ``` -```python -# Evaluator plugin execution through the plugin-owned SDK. -from nemo_evaluator.sdk.standalone_sdk.backend import AsyncNMPBackend - -client = AsyncNeMoPlatform(workspace="default") -evaluator = Evaluator(client=AsyncNMPBackend(client.evaluator)) -result = await evaluator.run(metrics=metric, dataset=data) -``` - ## Design Notes - `Evaluator()` uses `LocalBackend`. -- `Evaluator(client=...)` accepts an evaluator backend object. For NeMo Platform - execution, wrap the mounted evaluator resource in `NMPBackend` or - `AsyncNMPBackend` from the evaluator plugin package. -- `nemo_platform` and `nemo_evaluator` are optional for local SDK evaluation. +- `Evaluator(client=...)` accepts an evaluator backend object. Platform-specific + backend adapters are provided by + [`nemo-evaluator-plugin`](../../../../../plugins/nemo-evaluator/README.md). diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py index a0261250dd..2e214fe3d3 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py @@ -3,8 +3,6 @@ """Aggregation data structures and computations for metric results.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py - import math from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/templates.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/templates.py index 826e669749..2bf5b8b452 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/templates.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/templates.py @@ -3,8 +3,6 @@ """Template rendering helpers for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/templates.py - import json from typing import Any diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/common.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/common.py index 1930b94cf2..0fe2dcd2d7 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/common.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/common.py @@ -3,8 +3,6 @@ """Common value types used throughout evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/common.py - from enum import Enum from pydantic import Field, RootModel diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/datasets.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/datasets.py index 74eb725e9a..ae636c330d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/datasets.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/datasets.py @@ -3,8 +3,6 @@ """Dataset-related value types for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/datasets.py - from typing import Any, TypeAlias import pyarrow as pa diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index a629958d6d..b69e04a983 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -3,9 +3,6 @@ """Result types for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/results.py -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/scores.py - from __future__ import annotations import json diff --git a/packages/nemo_evaluator_sdk/tests/test_params.py b/packages/nemo_evaluator_sdk/tests/test_params.py index 5fac180999..8d9c7bce46 100644 --- a/packages/nemo_evaluator_sdk/tests/test_params.py +++ b/packages/nemo_evaluator_sdk/tests/test_params.py @@ -1,61 +1,45 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any - import pytest -from nemo_evaluator_sdk.values.params import ( - InferenceParams as SDKInferenceParams, -) -from nemo_evaluator_sdk.values.params import RunConfigOnlineModel -from nmp.common.inference import InferenceParams as CommonInferenceParams -from pydantic import BaseModel, ValidationError - - -def _normalize_field_contract(model: type[BaseModel]) -> dict[str, dict[str, Any]]: - return { - name: { - "annotation": field.annotation, - "default": field.default, - "default_factory": field.default_factory, - "alias": field.alias, - "validation_alias": field.validation_alias, - "serialization_alias": field.serialization_alias, - } - for name, field in model.model_fields.items() - } +from nemo_evaluator_sdk.values.params import InferenceParams, RunConfigOnlineModel +from pydantic import BaseModel, Field, ValidationError class TestInferenceParamsContract: - def test_inference_params_contract_matches_nmp_common(self): - expected_model_fields = CommonInferenceParams.model_fields - expected_model_fields.pop("model") - assert list(SDKInferenceParams.model_fields) == list(expected_model_fields) - assert _normalize_field_contract(SDKInferenceParams) == _normalize_field_contract(CommonInferenceParams) - assert dict(SDKInferenceParams.model_config) == dict(CommonInferenceParams.model_config) - assert callable(getattr(SDKInferenceParams, "check_max_tokens", None)) - assert callable(getattr(CommonInferenceParams, "check_max_tokens", None)) - - @pytest.mark.parametrize("model_cls", [SDKInferenceParams, CommonInferenceParams]) - def test_inference_params_enforces_same_token_limit_invariant(self, model_cls: type[BaseModel]): + def test_inference_params_exposes_expected_sdk_contract(self): + assert list(InferenceParams.model_fields) == [ + "temperature", + "max_tokens", + "max_completion_tokens", + "top_p", + "stop", + ] + assert InferenceParams.model_config.get("extra") == "allow" + assert callable(getattr(InferenceParams, "check_max_tokens", None)) + + def test_inference_params_enforces_token_limit_invariant(self): with pytest.raises(ValidationError, match="max_tokens and max_completion_tokens cannot both be configured"): - model_cls(max_tokens=1, max_completion_tokens=1) + InferenceParams(max_tokens=1, max_completion_tokens=1) class TestRunConfigOnlineModelInferenceCoercion: - def test_evaluation_job_params_coerces_nmp_common_inference_params(self): - common_params = CommonInferenceParams.model_validate( - { - "temperature": 0.25, - "max_tokens": 12, - "extra_body": {"nvext": {"max_thinking_tokens": 64}}, - } - ) - - params = RunConfigOnlineModel.model_validate({"inference": common_params}) - - assert isinstance(params.inference, SDKInferenceParams) - assert params.inference.model_dump(mode="python") == common_params.model_dump(mode="python", exclude={"model"}) + def test_evaluation_job_params_coerces_compatible_foreign_inference_model(self): + class CompatibleInferenceParams(BaseModel): + temperature: float = 0.25 + max_tokens: int = 12 + extra_body: dict[str, dict[str, int]] = Field( + default_factory=lambda: {"nvext": {"max_thinking_tokens": 64}} + ) + + params = RunConfigOnlineModel.model_validate({"inference": CompatibleInferenceParams()}) + + assert isinstance(params.inference, InferenceParams) + assert params.inference.model_dump(mode="python", exclude_none=True) == { + "temperature": 0.25, + "max_tokens": 12, + "extra_body": {"nvext": {"max_thinking_tokens": 64}}, + } def test_evaluation_job_params_rejects_incompatible_foreign_model(self): class ForeignInferenceParams(BaseModel): diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index 8809e1d1ad..112f73c5e3 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -330,7 +330,8 @@ nemo-evaluator-plugin = [ "cloudpickle>=3.1.1", "nemo-evaluator-sdk", "nemo-platform-plugin", - "nmp-evaluator", + "nemo-platform-sdk", + "nmp-common", "pydantic>=2.10.6", "typer>=0.20.0", ] diff --git a/packages/nemo_evaluator_sdk/examples/plugin_examples.py b/plugins/nemo-evaluator/examples/plugin_examples.py similarity index 99% rename from packages/nemo_evaluator_sdk/examples/plugin_examples.py rename to plugins/nemo-evaluator/examples/plugin_examples.py index 5a699e5e69..fd98c92ce5 100644 --- a/packages/nemo_evaluator_sdk/examples/plugin_examples.py +++ b/plugins/nemo-evaluator/examples/plugin_examples.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, cast from nemo_evaluator.jobs.evaluate import EvaluateSpec +from nemo_evaluator.sdk import FilesetRef from nemo_evaluator.sdk.resources import AsyncEvaluator from nemo_evaluator.sdk.resources import Evaluator as SyncEvaluator from nemo_evaluator.sdk.types import ( @@ -40,7 +41,6 @@ from nemo_evaluator_sdk.values.results import EvaluationResult from nemo_platform import APIError, AsyncNeMoPlatform, ConflictError, NeMoPlatform, NotFoundError from nemo_platform.types.files import HuggingfaceStorageConfigParam -from nmp.evaluator.app.values import FilesetRef if TYPE_CHECKING: import numpy as np diff --git a/plugins/nemo-evaluator/pyproject.toml b/plugins/nemo-evaluator/pyproject.toml index 8c5f9fb49d..b855b8eb83 100644 --- a/plugins/nemo-evaluator/pyproject.toml +++ b/plugins/nemo-evaluator/pyproject.toml @@ -8,8 +8,8 @@ dependencies = [ "cloudpickle>=3.1.1", "nemo-evaluator-sdk", "nemo-platform-plugin", - "nemo-platform", - "nmp-evaluator", + "nemo-platform-sdk", + "nmp-common", "pydantic>=2.10.6", "typer>=0.20.0", ] @@ -42,8 +42,8 @@ packages = ["src/nemo_evaluator"] [tool.uv.sources] nemo-evaluator-sdk = { workspace = true } nemo-platform-plugin = { workspace = true } -nemo-platform = { workspace = true } -nmp-evaluator = { workspace = true } +nemo-platform-sdk = { workspace = true } +nmp-common = { workspace = true } [dependency-groups] dev = ["pytest>=8.3.4", "pytest-asyncio>=0.25.3", "ruff>=0.11.8"] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index 07fab01dbb..c072ae07b3 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -12,6 +12,7 @@ from nemo_evaluator.jobs.utils import resolve_run_dataset from nemo_evaluator.resolvers import PlatformModelResolver +from nemo_evaluator.sdk.values.filesets import FilesetRef from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, unbundle_metric from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricPayload # noqa: F401 from nemo_evaluator_sdk import Evaluator @@ -32,7 +33,6 @@ from nemo_platform_plugin.job import NemoJob from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec -from nmp.evaluator.app.values import FilesetRef from pydantic import BaseModel, ConfigDict, Field, model_validator TargetSpec = Model | Agent diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py index c17c7eb0c0..0557bf8436 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/utils.py @@ -5,13 +5,153 @@ from __future__ import annotations -from typing import Any +from collections.abc import Sequence +from pathlib import Path +from typing import Any, cast +from nemo_evaluator.sdk.values.filesets import FilesetRef +from nemo_evaluator_sdk.datasets.loader import is_glob_pattern from nemo_evaluator_sdk.execution.metric_execution import run_sync +from nemo_evaluator_sdk.metrics.types import MetricsUnion +from nemo_evaluator_sdk.values import DatasetRows from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform.filesets import FilesetFileSystem, build_fileset_ref, parse_fileset_ref from nemo_platform_plugin.job_context import JobContext -from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync -from nmp.evaluator.app.values import FilesetRef + + +def _fileset_ref_parts(dataset: FilesetRef) -> tuple[str, str, str]: + """Return workspace, fileset name, and optional file selector.""" + return parse_fileset_ref(dataset.root, workspace_fallback=None) + + +def _base_download_path(destination: str, workspace: str, fileset: str) -> Path: + return Path(destination) / workspace / fileset + + +async def dataset_exists(sdk: AsyncNeMoPlatform, dataset: FilesetRef) -> bool: + """Return whether a persisted fileset reference resolves through the Files API.""" + fs = FilesetFileSystem(sdk=sdk) + workspace, fileset, path = _fileset_ref_parts(dataset) + base_path = build_fileset_ref("", workspace=workspace, fileset=fileset) + try: + if not path: + return await fs._exists(base_path) + if is_glob_pattern(path): + files_resource = cast(Any, sdk.files) + response = await files_resource.list(remote_path=path, fileset=fileset, workspace=workspace) + return bool(response.data) + return await fs._exists(build_fileset_ref(path, workspace=workspace, fileset=fileset)) + except FileNotFoundError: + return False + + +async def download_dataset( + *, + sdk: AsyncNeMoPlatform, + dataset: FilesetRef, + destination: str, +) -> Path: + """Download a persisted fileset reference to local storage.""" + workspace, fileset, path = _fileset_ref_parts(dataset) + base_dest = _base_download_path(destination, workspace, fileset) + base_dest.mkdir(parents=True, exist_ok=True) + files_resource = cast(Any, sdk.files) + + if not path: + await files_resource.download( + remote_path="", + fileset=fileset, + workspace=workspace, + local_path=str(base_dest), + max_workers=None, + ) + return base_dest + + if is_glob_pattern(path): + await files_resource.download( + remote_path=path, + fileset=fileset, + workspace=workspace, + local_path=str(base_dest), + max_workers=None, + ) + return base_dest + + local_path = base_dest / path + await files_resource.download( + remote_path=path, + fileset=fileset, + workspace=workspace, + local_path=str(local_path), + max_workers=None, + ) + return local_path + + +def download_dataset_sync( + *, + sdk: NeMoPlatform, + dataset: FilesetRef, + destination: str, +) -> Path: + """Sync bridge for fileset reference downloads.""" + workspace, fileset, path = _fileset_ref_parts(dataset) + base_dest = _base_download_path(destination, workspace, fileset) + base_dest.mkdir(parents=True, exist_ok=True) + files_resource = cast(Any, sdk.files) + + if not path: + files_resource.download( + remote_path="", + fileset=fileset, + workspace=workspace, + local_path=str(base_dest), + max_workers=None, + ) + return base_dest + + if is_glob_pattern(path): + files_resource.download( + remote_path=path, + fileset=fileset, + workspace=workspace, + local_path=str(base_dest), + max_workers=None, + ) + return base_dest + + local_path = base_dest / path + files_resource.download( + remote_path=path, + fileset=fileset, + workspace=workspace, + local_path=str(local_path), + max_workers=None, + ) + return local_path + + +def remote_compile_metric(metric: MetricsUnion | Sequence[MetricsUnion]) -> MetricsUnion: + """Return the single metric supported by evaluator plugin job compilation.""" + if isinstance(metric, Sequence): + raise NotImplementedError("Remote benchmark compilation is not implemented for inline evaluator plugin specs.") + return cast(MetricsUnion, metric) + + +async def resolve_submit_dataset( + async_sdk: AsyncNeMoPlatform, + dataset: list[dict[str, object]] | FilesetRef, +) -> tuple[DatasetRows | FilesetRef, FilesetRef | None]: + """Resolve an evaluator plugin dataset for remote metric-job submission. + + FilesetRef datasets are validated via the async SDK and passed through; + inline rows are wrapped as ``DatasetRows``. + """ + if isinstance(dataset, FilesetRef): + if not await dataset_exists(async_sdk, dataset): + raise ValueError(f"FilesetRef dataset does not exist: {dataset.root}") + return dataset, dataset + return DatasetRows(rows=dataset), None def resolve_run_dataset( diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/__init__.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/__init__.py index edd78023bc..8ee0b9e2f6 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/__init__.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/__init__.py @@ -3,16 +3,20 @@ """Public Evaluator plugin SDK surface.""" -from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource -from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator +from typing import TYPE_CHECKING + from nemo_evaluator.sdk.types import ( ExecutionMode, - FilesetRef, PluginDatasetInput, RunConfig, RunConfigOnline, RunConfigOnlineModel, ) +from nemo_evaluator.sdk.values.filesets import FilesetRef + +if TYPE_CHECKING: + from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource + from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator __all__ = [ "AsyncEvaluator", @@ -26,3 +30,32 @@ "FilesetRef", "PluginDatasetInput", ] + + +def __getattr__(name: str) -> object: + """Load resource classes only when callers access them. + + Keep ``import nemo_evaluator.sdk`` lightweight for examples and SDK users + that only need value types such as ``FilesetRef`` or ``RunConfig``. Eagerly + importing ``resources`` pulls in ``sdk._executor``, which imports + ``jobs.evaluate`` and the plugin job/metric-bundle runtime. The lightweight + path is used by imports such as ``from nemo_evaluator.sdk import FilesetRef``; + preserve this lazy boundary so those imports do not initialize job code. + """ + if name == "AsyncEvaluator": + from nemo_evaluator.sdk.resources import AsyncEvaluator + + return AsyncEvaluator + if name == "Evaluator": + from nemo_evaluator.sdk.resources import Evaluator + + return Evaluator + if name == "AsyncEvaluatorJobResource": + from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource + + return AsyncEvaluatorJobResource + if name == "EvaluatorJobResource": + from nemo_evaluator.sdk.job_resources import EvaluatorJobResource + + return EvaluatorJobResource + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index ff192a29ba..e4042028f4 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -14,6 +14,7 @@ import httpx from nemo_evaluator.jobs.evaluate import EvaluateJob, EvaluateSpec +from nemo_evaluator.jobs.utils import download_dataset, download_dataset_sync from nemo_evaluator.sdk import http_utils from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult from nemo_evaluator.sdk.job_resources import ( @@ -23,6 +24,7 @@ ) from nemo_evaluator.sdk.types import PluginDatasetInput from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result +from nemo_evaluator.sdk.values.filesets import FilesetRef from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, MetricBundlePackager, bundle_metric from nemo_evaluator_sdk import Evaluator as SDKEvaluator from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows @@ -40,8 +42,6 @@ from nemo_evaluator_sdk.values.results import AggregateFieldName, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.scheduler import NemoJobScheduler -from nmp.evaluator.app.datasets.nmp_datasets.fileset import download_dataset, download_dataset_sync -from nmp.evaluator.app.values import FilesetRef _DEFAULT_POLL_INTERVAL_SECONDS = 10.0 _DEFAULT_JOB_TIMEOUT_SECONDS = 3600.0 diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/types.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/types.py index 4e672dcfc5..768b881bb1 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/types.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/types.py @@ -8,13 +8,13 @@ from pathlib import Path from typing import Literal, TypeAlias +from nemo_evaluator.sdk.values.filesets import FilesetRef from nemo_evaluator_sdk.values import ( DatasetInput, RunConfig, RunConfigOnline, RunConfigOnlineModel, ) -from nmp.evaluator.app.values import FilesetRef # TODO: remove this type if we decide nemo_evaluator_sdk will not support remote execution. ExecutionMode: TypeAlias = Literal["local", "remote"] @@ -24,6 +24,7 @@ "RunConfig", "RunConfigOnline", "RunConfigOnlineModel", + "FilesetRef", "ExecutionMode", "PluginDatasetInput", ] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/values/filesets.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/values/filesets.py new file mode 100644 index 0000000000..30256b2262 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/values/filesets.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fileset value objects for the evaluator plugin SDK.""" + +from __future__ import annotations + +from pydantic import Field, RootModel + + +class FilesetRef(RootModel[str]): + """Reference to a Fileset in the platform Files API. + + A reference is a string with format 'workspace/fileset-name' that points to a + persisted fileset entity. When used as a dataset source, all files within the + fileset will be downloaded to the job container. + """ + + root: str = Field(description="Reference to a Fileset (format: workspace/fileset-name).") + + def with_fragment(self, fragment: str) -> FilesetRef: + """Return a new fileset reference with a file path fragment appended.""" + normalized_fragment = fragment.lstrip("/") + if not normalized_fragment: + raise ValueError("FilesetRef fragment cannot be empty.") + if "#" in normalized_fragment: + raise ValueError("FilesetRef fragment cannot contain '#'.") + if "#" in self.root: + raise ValueError("FilesetRef already includes a fragment.") + return FilesetRef(root=f"{self.root}#{normalized_fragment}") + + +__all__ = ["FilesetRef"] diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py index 8bacacd2a2..ee87134856 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/tasks/evaluate.py @@ -5,15 +5,40 @@ from __future__ import annotations +import logging +import signal +import sys +from enum import IntEnum +from types import FrameType + from nemo_evaluator.jobs.evaluate import EvaluateJob from nemo_platform_plugin.tasks.dispatcher import run_task from nmp.common.sdk_factory import get_task_sdk +logger = logging.getLogger(__name__) + + +class EvaluateTaskExitCode(IntEnum): + """Evaluator task setup exit codes.""" + + SDK_INITIALIZATION_FAILED = 2 + + +def _shutdown_handler(signum: int, frame: FrameType | None) -> None: + logger.warning("Received shutdown signal (%d). Exiting.", signum) + raise SystemExit(128 + signum) + def main() -> int: - """Run the evaluator job in a platform-spawned task process.""" - return run_task(EvaluateJob, sdk=get_task_sdk("evaluator")) + """Build the task SDK and dispatch to the evaluator plugin job.""" + signal.signal(signal.SIGTERM, _shutdown_handler) + try: + sdk = get_task_sdk("evaluator") + except Exception: + logger.exception("Failed to build task SDK for evaluator") + return EvaluateTaskExitCode.SDK_INITIALIZATION_FAILED + return run_task(EvaluateJob, sdk=sdk) if __name__ == "__main__": - raise SystemExit(main()) + sys.exit(main()) diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index 0faf03737b..e7b9f5ff93 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -22,6 +22,7 @@ EvaluateSpec, ) from nemo_evaluator.resolvers import PlatformModelResolver, _parse_required_workspace_name +from nemo_evaluator.sdk.values.filesets import FilesetRef from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, MetricBundlePackager, @@ -31,6 +32,7 @@ unbundle_metric, ) from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager +from nemo_evaluator.tasks.evaluate import EvaluateTaskExitCode from nemo_evaluator.tasks.evaluate import main as evaluate_task_main from nemo_evaluator_sdk.enums import AgentFormat from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric @@ -55,7 +57,6 @@ from nemo_platform_plugin.job_results import LocalJobResults from nemo_platform_plugin.jobs.constants import PERSISTENT_JOB_STORAGE_PATH_ENVVAR from nemo_platform_plugin.scheduler import NemoJobScheduler -from nmp.evaluator.app.values import FilesetRef from pydantic import BaseModel, ConfigDict from pytest_mock import MockerFixture from typer.testing import CliRunner @@ -974,3 +975,16 @@ def test_main_dispatches_evaluate_job_with_task_sdk(self, mocker: MockerFixture) assert exit_code == 0 get_task_sdk.assert_called_once_with("evaluator") run_task.assert_called_once_with(EvaluateJob, sdk=sdk) + + def test_main_returns_setup_exit_code_when_task_sdk_fails(self, mocker: MockerFixture) -> None: + get_task_sdk = mocker.patch( + "nemo_evaluator.tasks.evaluate.get_task_sdk", + side_effect=RuntimeError("boom"), + ) + run_task = mocker.patch("nemo_evaluator.tasks.evaluate.run_task") + + exit_code = evaluate_task_main() + + assert exit_code == EvaluateTaskExitCode.SDK_INITIALIZATION_FAILED + get_task_sdk.assert_called_once_with("evaluator") + run_task.assert_not_called() diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py index 93b6e5dbb5..4bc19ba921 100644 --- a/plugins/nemo-evaluator/tests/test_sdk.py +++ b/plugins/nemo-evaluator/tests/test_sdk.py @@ -23,6 +23,7 @@ from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator +from nemo_evaluator.sdk.values.filesets import FilesetRef from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, MetricBundlePackager, @@ -38,7 +39,6 @@ from nemo_evaluator_sdk.values.results import AggregatedMetricResult, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.jobs.schemas import PlatformJobStatus -from nmp.evaluator.app.values import FilesetRef from pydantic import ValidationError from pytest_mock import MockerFixture diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py index eae87da00f..14409d42a4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/datasets/loader.py @@ -3,8 +3,6 @@ """Dataset loader module for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/datasets/loader.py - import gzip import io import json diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py index 90523d7793..d802bf45f7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py @@ -3,8 +3,6 @@ """Aggregation data structures and computations for metric results.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py - import math from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py index 826e669749..2bf5b8b452 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/templates.py @@ -3,8 +3,6 @@ """Template rendering helpers for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/templates.py - import json from typing import Any diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py index 1930b94cf2..0fe2dcd2d7 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/common.py @@ -3,8 +3,6 @@ """Common value types used throughout evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/common.py - from enum import Enum from pydantic import Field, RootModel diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py index 74eb725e9a..ae636c330d 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/datasets.py @@ -3,8 +3,6 @@ """Dataset-related value types for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/datasets.py - from typing import Any, TypeAlias import pyarrow as pa diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index 47c216a873..3157bc5685 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -3,9 +3,6 @@ """Result types for evaluator SDK runtime.""" -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/results.py -# Migrated from: services/evaluator/src/nmp/evaluator/app/values/scores.py - from __future__ import annotations import json diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/vendor/vendor_package.py b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/vendor/vendor_package.py index 3ca9d23198..a1edd2945e 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/vendor/vendor_package.py +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/vendor/vendor_package.py @@ -586,6 +586,21 @@ def _should_copy_optional_dependency_extra(extra_name: str, target_project_name: return True +def _generated_optional_dependency_groups(optional: tomlkit.items.Table) -> set[str]: + """Return optional-dependency groups owned by the bundle generator.""" + generated: set[str] = set() + saw_marker = False + for key, item in optional._value.body: + if key is None: + if isinstance(item, tomlkit.items.Comment) and item.as_string().strip() == GENERATED_BUNDLE_GROUP_COMMENT: + saw_marker = True + continue + if saw_marker: + generated.add(key.key) + saw_marker = False + return generated + + def _merge_matching_project_optional_dependencies( target_project: dict, source_project: dict, patterns: list[str] ) -> None: @@ -707,6 +722,7 @@ def _process_bundle_packages() -> None: pyproject = tomlkit.loads(member_pyproject_path.read_text(encoding="utf-8")) optional = pyproject["project"].setdefault("optional-dependencies", tomlkit.table()) + generated_optional_groups = _generated_optional_dependency_groups(optional) for pkg_name, pkg_config in bundle_config.items(): deps_group = _bundle_deps_group(pkg_name, pkg_config) @@ -759,7 +775,11 @@ def _process_bundle_packages() -> None: self_ref_deps = [d for d in filtered_deps if d.startswith(self_ref_prefix)] existing = list(optional.get(deps_group, [])) - existing_regular = [d for d in existing if not d.startswith(self_ref_prefix)] + existing_regular = ( + [] + if deps_group in generated_optional_groups + else [d for d in existing if not d.startswith(self_ref_prefix)] + ) existing_self_refs = [d for d in existing if d.startswith(self_ref_prefix)] merged_regular = merge_dependencies(existing_regular, regular_deps) diff --git a/tools/nemo-platform-sdk-tools/tests/sdk/vendor/test_vendor_package.py b/tools/nemo-platform-sdk-tools/tests/sdk/vendor/test_vendor_package.py index 1d1285969d..9df3de6a9a 100644 --- a/tools/nemo-platform-sdk-tools/tests/sdk/vendor/test_vendor_package.py +++ b/tools/nemo-platform-sdk-tools/tests/sdk/vendor/test_vendor_package.py @@ -323,6 +323,71 @@ def test_process_bundle_packages_keeps_bundled_dependency_names_without_inheriti assert not wrapper_updated["project"]["entry-points"] +def test_process_bundle_packages_rebuilds_generated_dependency_groups(tmp_path: Path, monkeypatch) -> None: + wrapper_path = tmp_path / "packages/nemo_platform" + plugin_path = tmp_path / "plugins/nemo-evaluator" + runner_path = tmp_path / "packages/nmp_platform_runner" + (plugin_path / "src/nemo_evaluator").mkdir(parents=True) + (runner_path / "src/nmp/platform_runner").mkdir(parents=True) + wrapper_path.mkdir(parents=True) + + (tmp_path / "pyproject.toml").write_text( + """ +[tool.uv.workspace] +members = ["packages/nemo_platform"] +""".lstrip(), + encoding="utf-8", + ) + (wrapper_path / "pyproject.toml").write_text( + """ +[project] +name = "nemo-platform" + +[project.optional-dependencies] +# Generated from [tool.bundle-package]; do not edit by hand. +nemo-evaluator-plugin = ["nemo-evaluator-sdk", "nmp-evaluator"] + +# Generated from [tool.bundle-package]; do not edit by hand. +services = ["nemo-platform[core-service]", "old-service"] + +[tool.bundle-package] +nemo-evaluator-plugin = { source = "../../plugins/nemo-evaluator/src/nemo_evaluator", module = "nemo_evaluator" } +nmp-platform-runner = { source = "../../packages/nmp_platform_runner/src/nmp/platform_runner", module = "nmp/platform_runner", deps_group = "services" } +""".lstrip(), + encoding="utf-8", + ) + (plugin_path / "pyproject.toml").write_text( + """ +[project] +name = "nemo-evaluator-plugin" +dependencies = ["nemo-evaluator-sdk", "nemo-platform-sdk", "nmp-common", "pydantic>=2.10.6"] +""".lstrip(), + encoding="utf-8", + ) + (runner_path / "pyproject.toml").write_text( + """ +[project] +name = "nmp-platform-runner" +dependencies = ["rich>=14.1.0"] +""".lstrip(), + encoding="utf-8", + ) + + monkeypatch.setattr(vendor_package, "NMP_ROOT_PATH", tmp_path) + vendor_package._process_bundle_packages() + + wrapper_updated = tomlkit.parse((wrapper_path / "pyproject.toml").read_text(encoding="utf-8")) + optional = wrapper_updated["project"]["optional-dependencies"] + + assert list(optional["nemo-evaluator-plugin"]) == [ + "nemo-evaluator-sdk", + "nemo-platform-sdk", + "nmp-common", + "pydantic>=2.10.6", + ] + assert list(optional["services"]) == ["rich>=14.1.0", "nemo-platform[core-service]"] + + def test_process_bundle_packages_inherits_requested_metadata(tmp_path: Path, monkeypatch) -> None: wrapper_path = tmp_path / "packages/nemo_platform" plugin_path = tmp_path / "plugins/nemo-switchyard" diff --git a/uv.lock b/uv.lock index 82c2fec643..e682fbb541 100644 --- a/uv.lock +++ b/uv.lock @@ -3813,9 +3813,9 @@ source = { editable = "plugins/nemo-evaluator" } dependencies = [ { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-evaluator-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nemo-platform", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-platform-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "typer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] @@ -3831,9 +3831,9 @@ dev = [ requires-dist = [ { name = "cloudpickle", specifier = ">=3.1.1" }, { name = "nemo-evaluator-sdk", editable = "packages/nemo_evaluator_sdk" }, - { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, - { name = "nmp-evaluator", editable = "services/evaluator" }, + { name = "nemo-platform-sdk", editable = "sdk/python/nemo-platform" }, + { name = "nmp-common", editable = "packages/nmp_common" }, { name = "pydantic", specifier = ">=2.10.6" }, { name = "typer", specifier = ">=0.20.0" }, ] @@ -4003,7 +4003,6 @@ all = [ { name = "nemoguardrails", extra = ["tracing"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "ngcsdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-config-optimizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4304,7 +4303,8 @@ nemo-evaluator-plugin = [ { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-evaluator-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-platform-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pydantic", extra = ["email"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "typer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] @@ -4410,10 +4410,10 @@ plugins = [ { name = "nemo-anonymizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-evaluator-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-platform-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-safe-synthesizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemoguardrails", extra = ["tracing"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-config-optimizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4477,7 +4477,6 @@ services = [ { name = "nemoguardrails", extra = ["tracing"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "ngcsdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "nmp-evaluator", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-config-optimizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nvidia-nat-langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -4790,8 +4789,10 @@ requires-dist = [ { name = "nemo-platform-sdk", marker = "extra == 'all'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'core-service'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'entities-service'", editable = "sdk/python/nemo-platform" }, + { name = "nemo-platform-sdk", marker = "extra == 'nemo-evaluator-plugin'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'nemo-platform-plugin'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'nmp-common'", editable = "sdk/python/nemo-platform" }, + { name = "nemo-platform-sdk", marker = "extra == 'plugins'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'services'", editable = "sdk/python/nemo-platform" }, { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.1" }, { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.1" }, @@ -4826,15 +4827,12 @@ requires-dist = [ { name = "nmp-common", marker = "extra == 'nemo-agents-plugin'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'nemo-anonymizer-plugin'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'nemo-data-designer-plugin'", editable = "packages/nmp_common" }, + { name = "nmp-common", marker = "extra == 'nemo-evaluator-plugin'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'nemo-safe-synthesizer-plugin'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'plugins'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'secrets-service'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'services'", editable = "packages/nmp_common" }, { name = "nmp-common", marker = "extra == 'studio-service'", editable = "packages/nmp_common" }, - { name = "nmp-evaluator", marker = "extra == 'all'", editable = "services/evaluator" }, - { name = "nmp-evaluator", marker = "extra == 'nemo-evaluator-plugin'", editable = "services/evaluator" }, - { name = "nmp-evaluator", marker = "extra == 'plugins'", editable = "services/evaluator" }, - { name = "nmp-evaluator", marker = "extra == 'services'", editable = "services/evaluator" }, { name = "nvidia-ml-py", marker = "extra == 'nemo-platform-sdk'", specifier = ">=13.0.0" }, { name = "nvidia-ml-py", marker = "extra == 'nmp-common'", specifier = ">=13.0.0" }, { name = "nvidia-nat-config-optimizer", marker = "extra == 'all'", specifier = ">=1.7.0,<1.8" },