diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/config.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/config.py index f3ec2ea2eb..ecd5c5f2db 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/config.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/execution/config.py @@ -24,6 +24,10 @@ def resolve_params( ) -> _RunConfigT: """Return params after validating that they match the selected target mode.""" if isinstance(target, Model): + if params is None or type(params) is RunConfig: + raise TypeError("model target requires RunConfigOnlineModel") + if type(params) is RunConfigOnline: + return RunConfigOnlineModel.model_validate(params.model_dump()) if not isinstance(params, RunConfigOnlineModel): raise TypeError("model target requires RunConfigOnlineModel") return params diff --git a/packages/nemo_evaluator_sdk/tests/execution/test_config.py b/packages/nemo_evaluator_sdk/tests/execution/test_config.py index af72cfa910..06fadb33cc 100644 --- a/packages/nemo_evaluator_sdk/tests/execution/test_config.py +++ b/packages/nemo_evaluator_sdk/tests/execution/test_config.py @@ -49,6 +49,17 @@ def test_accepts_model_online_params(self) -> None: assert resolve_params(params=params, target=target) is params + def test_converts_generic_online_params_for_model_target(self) -> None: + """Model targets accept generic online params from JSON specs.""" + target = Model(url="http://example.test/v1", name="test-model") + params = RunConfigOnline(parallelism=3, ignore_request_failure=True) + + resolved = resolve_params(params=params, target=target) + + assert isinstance(resolved, RunConfigOnlineModel) + assert resolved.parallelism == 3 + assert resolved.ignore_request_failure is True + def test_accepts_agent_online_params(self) -> None: """Agent targets require RunConfigOnline-compatible params.""" target = Agent( diff --git a/plugins/nemo-evaluator/examples/plugin_examples.py b/plugins/nemo-evaluator/examples/plugin_examples.py index fd98c92ce5..f5430757db 100644 --- a/plugins/nemo-evaluator/examples/plugin_examples.py +++ b/plugins/nemo-evaluator/examples/plugin_examples.py @@ -5,6 +5,7 @@ from __future__ import annotations +import argparse import asyncio import gzip import json @@ -13,14 +14,13 @@ from collections.abc import Sequence from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Literal, cast from nemo_evaluator.jobs.evaluate import EvaluateSpec from nemo_evaluator.sdk import FilesetRef from nemo_evaluator.sdk.resources import AsyncEvaluator from nemo_evaluator.sdk.resources import Evaluator as SyncEvaluator from nemo_evaluator.sdk.types import ( - ExecutionMode, PluginDatasetInput, RunConfig, RunConfigOnlineModel, @@ -56,6 +56,7 @@ 'Return only a JSON object with this shape: {"helpfulness": }.' ) ONLINE_CHAT_PROMPT_TEMPLATE = {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} +ExampleExecutionMode = Literal["run", "submit"] LOCAL_HELPSTEER2_ROWS = ( { "prompt": "What is the capital of France?", @@ -111,7 +112,8 @@ async def _new_client() -> AsyncNeMoPlatform: except APIError as e: await _close_client(client) raise RuntimeError( - f"Failed to connect to evaluator plugin. Ensure nemo-evaluator plugin is running along with `nemo services run`. Error: {e}" + "Failed to connect to evaluator plugin. Ensure nemo-evaluator plugin is running along with " + f"`nemo services run`. Error: {e}" ) from None return client @@ -128,7 +130,8 @@ def _new_sync_client() -> NeMoPlatform: except APIError as e: client.close() raise RuntimeError( - f"Failed to connect to evaluator plugin. Ensure nemo-evaluator plugin is running along with `nemo services run`. Error: {e}" + "Failed to connect to evaluator plugin. Ensure nemo-evaluator plugin is running along with " + f"`nemo services run`. Error: {e}" ) from None return client @@ -233,19 +236,19 @@ def ensure_example_fileset_sync(client: NeMoPlatform) -> FilesetRef: return FilesetRef(root=f"{fileset.workspace}/{fileset.name}").with_fragment(HELPSTEER2_REMOTE_PATH) -async def ensure_remote_evaluator_api_key_secret(workspace: str, client: AsyncNeMoPlatform) -> str: +async def ensure_submit_evaluator_api_key_secret(workspace: str, client: AsyncNeMoPlatform) -> str: """Resolve an API key secret name and ensure it exists on the platform.""" - secret_name = DEFAULT_API_KEY_SECRET.lower() + secret_name = DEFAULT_API_KEY_SECRET.lower().replace("_", "-") try: await client.secrets.retrieve(secret_name, workspace=workspace) except NotFoundError: api_key = os.getenv(DEFAULT_API_KEY_SECRET) or os.getenv("NVIDIA_API_KEY") or os.getenv("NVIDIA_BUILD_API_KEY") if api_key is None: raise RuntimeError( - f"Remote online evaluation needs a platform secret named '{secret_name}' in workspace " + f"Submit-mode online evaluation needs a platform secret named '{secret_name}' in workspace " f"'{workspace}'. Set NVIDIA_BUILD_API_KEY or NVIDIA_API_KEY to let this " "example create it, or create it manually with: " - f"nemo secrets create {secret_name} --data '' --workspace {workspace}" + f"nemo secrets create {secret_name} --value '' --workspace {workspace}" ) from None try: await client.secrets.create(workspace=workspace, name=secret_name, value=api_key) @@ -257,13 +260,13 @@ async def ensure_remote_evaluator_api_key_secret(workspace: str, client: AsyncNe async def model_with_valid_secret( *, - execution_mode: ExecutionMode, + execution_mode: ExampleExecutionMode, workspace: str, client: AsyncNeMoPlatform, ) -> Model: - """Return a model configured for local or remote NeMo Platform example execution.""" - if execution_mode == "remote": - secret_name = await ensure_remote_evaluator_api_key_secret(workspace, client) + """Return a model configured for run or submit NeMo Platform example execution.""" + if execution_mode == "submit": + secret_name = await ensure_submit_evaluator_api_key_secret(workspace, client) return model.model_copy(update={"api_key_secret": SecretRef(root=secret_name)}) return model @@ -366,14 +369,14 @@ def _assert_exact_match_result(result: EvaluationResult, *, workflow: str, expec async def _evaluate_metric( evaluator_plugin_client: AsyncEvaluator, *, - execution_mode: ExecutionMode, + execution_mode: ExampleExecutionMode, metric: Metric, dataset: PluginDatasetInput, config: RunConfig | RunConfigOnlineModel, **run_kwargs: Any, ) -> EvaluationResult: - """Run locally or submit remotely based on the requested plugin SDK execution mode.""" - if execution_mode == "local": + """Run or submit based on the requested plugin SDK execution mode.""" + if execution_mode == "run": return await evaluator_plugin_client.run( metric=metric, dataset=dataset, @@ -440,7 +443,7 @@ async def _run_online_metric_example_body( dataset: PluginDatasetInput, workflow_label: str, is_online: bool, - execution_mode: ExecutionMode, + execution_mode: ExampleExecutionMode, limit_samples: int, ) -> None: """Evaluate one exact-match metric against an already-built dataset. @@ -484,10 +487,10 @@ async def _run_online_metric_example_body( async def run_nmp_online_metric_example( is_online: bool = False, - execution_mode: ExecutionMode = "local", + execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: - """Evaluate one metric through the plugin SDK using local run or remote submit.""" + """Evaluate one metric through the plugin SDK using run or submit.""" _print_example_separator( run_nmp_online_metric_example.__name__, is_online=is_online, @@ -511,7 +514,7 @@ async def run_nmp_online_metric_example( def run_nmp_online_metric_example_sync_client( is_online: bool = False, - execution_mode: ExecutionMode = "local", + execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: """Evaluate one metric through the plugin SDK using a sync platform client.""" @@ -535,7 +538,7 @@ def run_nmp_online_metric_example_sync_client( run_kwargs["target"] = model run_kwargs["prompt_template"] = ONLINE_CHAT_PROMPT_TEMPLATE - if execution_mode == "local": + if execution_mode == "run": result = evaluator_plugin_client.run( metric=metric, dataset=dataset, @@ -574,7 +577,7 @@ def run_nmp_online_metric_example_sync_client( async def run_nmp_online_metric_local_file_example( is_online: bool = False, - execution_mode: ExecutionMode = "local", + execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: """Evaluate one metric through the plugin SDK using a local JSONL Path dataset.""" @@ -604,9 +607,9 @@ async def run_nmp_online_metric_local_file_example( async def run_nmp_llm_judge_example( is_online: bool = False, limit_samples: int = 2, - execution_mode: ExecutionMode = "local", + execution_mode: ExampleExecutionMode = "run", ) -> None: - """Evaluate a helpfulness judge through the plugin SDK using local run or remote submit.""" + """Evaluate a helpfulness judge through the plugin SDK using run or submit.""" _print_example_separator( run_nmp_llm_judge_example.__name__, is_online=is_online, @@ -653,22 +656,61 @@ async def run_nmp_llm_judge_example( print(f"human avg: {human_scores.mean()}") -async def run_examples() -> None: +async def run_examples(*, include_submit: bool = False, include_model_calls: bool = False) -> None: """Execute the example workflows exposed by this module.""" - await run_nmp_online_metric_example(is_online=False, execution_mode="local") - await run_nmp_online_metric_example(is_online=False, execution_mode="remote") - await run_nmp_llm_judge_example(is_online=False, execution_mode="local") - await run_nmp_llm_judge_example(is_online=True, execution_mode="remote") - await run_nmp_online_metric_local_file_example(is_online=False, execution_mode="local") + await run_nmp_online_metric_example(is_online=False, execution_mode="run") + await run_nmp_online_metric_local_file_example(is_online=False, execution_mode="run") + if include_submit: + await run_nmp_online_metric_example(is_online=False, execution_mode="submit") -def run_sync_examples() -> None: + if include_model_calls: + await run_nmp_llm_judge_example(is_online=False, execution_mode="run") + if include_submit: + await run_nmp_llm_judge_example(is_online=True, execution_mode="submit") + + +def run_sync_examples(*, include_submit: bool = False) -> None: """Execute the synchronous example workflows exposed by this module.""" - run_nmp_online_metric_example_sync_client(is_online=False, execution_mode="remote") + if include_submit: + run_nmp_online_metric_example_sync_client(is_online=False, execution_mode="submit") -if __name__ == "__main__": +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse example runner options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--include-submit", + action="store_true", + help="Submit evaluator jobs in addition to run-mode examples.", + ) + parser.add_argument( + "--include-model-calls", + action="store_true", + help="Run judge or online target examples that call hosted models.", + ) + parser.add_argument( + "--include-sync-submit", + action="store_true", + help="Run the synchronous submit example.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run examples with safe defaults.""" + args = parse_args(argv) configure_example_logging() - run_sync_examples() - asyncio.run(run_examples()) + run_sync_examples(include_submit=args.include_sync_submit) + asyncio.run( + run_examples( + include_submit=args.include_submit, + include_model_calls=args.include_model_calls, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py index ae44acc22b..d74638d0e9 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py @@ -5,11 +5,80 @@ from __future__ import annotations +import inspect import json -from typing import ClassVar +from enum import Enum +from types import UnionType +from typing import Annotated, Any, ClassVar, Union, get_args, get_origin import typer +from nemo_evaluator_sdk.metrics.types import MetricVariants +from nemo_evaluator_sdk.values.metrics import _RAGASBase from nemo_platform_plugin.cli import NemoCLI +from pydantic import BaseModel + + +def _unwrap_metric_model_classes(type_hint: object) -> list[type[BaseModel]]: + """Return Pydantic model classes from an annotated metric union.""" + origin = get_origin(type_hint) + if origin is Annotated: + return _unwrap_metric_model_classes(get_args(type_hint)[0]) + if origin in {Union, UnionType}: + model_classes: list[type[BaseModel]] = [] + for union_member in get_args(type_hint): + model_classes.extend(_unwrap_metric_model_classes(union_member)) + return model_classes + if isinstance(type_hint, type) and issubclass(type_hint, BaseModel): + return [type_hint] + return [] + + +def _json_value(value: object) -> object: + if isinstance(value, Enum): + return value.value + return value + + +def _metric_type_values(model_cls: type[BaseModel]) -> list[str]: + type_field = model_cls.model_fields["type"] + annotation_args = get_args(type_field.annotation) + if annotation_args: + return [str(_json_value(value)) for value in annotation_args] + return [str(_json_value(type_field.default))] + + +def _metric_type_models() -> dict[str, type[BaseModel]]: + metric_types: dict[str, type[BaseModel]] = {} + for model_cls in _unwrap_metric_model_classes(MetricVariants): + for metric_type in _metric_type_values(model_cls): + existing = metric_types.get(metric_type) + if existing is not None and existing is not model_cls: + raise ValueError( + f"Duplicate metric type '{metric_type}' mapped to both {existing.__name__} and {model_cls.__name__}" + ) + metric_types[metric_type] = model_cls + return dict(sorted(metric_types.items())) + + +def _is_ragas_metric(model_cls: type[BaseModel]) -> bool: + return issubclass(model_cls, _RAGASBase) + + +def _metric_type_entries() -> list[dict[str, str]]: + return [ + { + "name": metric_type, + "description": inspect.getdoc(model_cls) or "", + } + for metric_type, model_cls in sorted( + _metric_type_models().items(), + key=lambda item: (_is_ragas_metric(item[1]), item[0]), + ) + ] + + +def _echo_json(payload: Any) -> None: + typer.echo(json.dumps(payload, indent=2)) class EvaluatorPluginCLI(NemoCLI): @@ -28,17 +97,33 @@ def get_cli(self) -> typer.Typer: @app.command("info") def info() -> None: """Print the current plugin status.""" - typer.echo( - json.dumps( - { - "plugin": self.name, - "status": "ready", - "service": "/apis/evaluator/v1/healthz", - "jobs": ["evaluator.evaluate"], - "sdk": "nemo_evaluator_sdk.Evaluator", - }, - indent=2, - ) + _echo_json( + { + "plugin": self.name, + "status": "ready", + "service": "/apis/evaluator/v1/healthz", + "jobs": ["evaluator.evaluate"], + "sdk": "nemo_evaluator_sdk.Evaluator", + } ) + @app.command("metric-types") + def metric_types( + metric_types_name: str | None = typer.Argument(None, metavar=""), + ) -> None: + """Print available evaluator metric names or a metric JSON schema.""" + if metric_types_name is None: + _echo_json({"metric_types": _metric_type_entries()}) + return + + metric_types_map = _metric_type_models() + model_cls = metric_types_map.get(metric_types_name) + if model_cls is None: + typer.echo( + f"Unknown metric name '{metric_types_name}'. Run `nemo evaluator metric-types` to list available metric names.", + err=True, + ) + raise typer.Exit(code=1) + _echo_json(model_cls.model_json_schema()) + return app diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data b/plugins/nemo-evaluator/src/nemo_evaluator/docs/data new file mode 120000 index 0000000000..209cb8f6c5 --- /dev/null +++ b/plugins/nemo-evaluator/src/nemo_evaluator/docs/data @@ -0,0 +1 @@ +../../../../../skills/nemo-evaluator-plugin/assets/specs \ No newline at end of file diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_benchmark.json b/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_benchmark.json deleted file mode 100644 index 6c6a61e86b..0000000000 --- a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_benchmark.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "metric": [ - { - "type": "exact-match", - "labels": {}, - "reference": "{{item.reference}}" - }, - { - "type": "string-check", - "labels": {}, - "operation": "contains", - "left_template": "{{sample.output_text}}", - "right_template": "{{item.required_phrase}}" - } - ], - "dataset": [ - { - "prompt": "Return exactly this word with no punctuation: Paris", - "reference": "Paris", - "required_phrase": "Paris" - }, - { - "note": "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are 'London' so both metrics should report a miss.", - "prompt": "Return exactly this word with no punctuation: Oslo", - "reference": "London", - "required_phrase": "London" - } - ], - "params": { - "parallelism": 4, - "limit_samples": 2, - "ignore_request_failure": false, - "request_timeout": 60, - "max_retries": 3 - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim" - }, - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "{{item.prompt}}" - } - ] - } -} \ No newline at end of file diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json b/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json deleted file mode 100644 index 6f8a32356c..0000000000 --- a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "metric": { - "type": "exact-match", - "reference": "{{item.expected}}", - "candidate": "{{item.model_output}}" - }, - "dataset": [ - { - "expected": "blue", - "model_output": "Blue" - }, - { - "expected": "Jupiter", - "model_output": "Saturn" - } - ], - "params": { - "parallelism": 2 - } -} \ No newline at end of file diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/llm_as_judge.json b/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/llm_as_judge.json deleted file mode 100644 index 4523719c41..0000000000 --- a/plugins/nemo-evaluator/src/nemo_evaluator/docs/data/llm_as_judge.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "metric": { - "type": "llm-judge", - "labels": {}, - "model": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim" - }, - "scores": [ - { - "name": "helpfulness", - "description": "How well does the response help the user?", - "parser": { - "type": "json", - "json_path": "helpfulness" - }, - "minimum": 0, - "maximum": 4 - } - ], - "prompt_template": { - "messages": [ - { - "role": "system", - "content": "You are an evaluator. Rate the response's helpfulness from 0-4. Return only a JSON object with this shape: {\"helpfulness\": }." - }, - { - "role": "user", - "content": "User prompt: {{item.input}}\n\nAssistant response: {{sample.output_text | default(item.output)}}\n\nRate this response." - } - ] - }, - "optional_fields": [], - "structured_output": { - "schema": { - "type": "object", - "properties": { - "helpfulness": { - "type": "integer", - "minimum": 0, - "maximum": 4 - } - }, - "required": [ - "helpfulness" - ] - } - }, - "inference": { - "temperature": 0.0, - "max_tokens": 32768, - "max_completion_tokens": null, - "top_p": null, - "stop": null - }, - "system_prompt": null, - "reasoning": null, - "ignore_request_failure": false, - "job_type": "online" - }, - "dataset": [ - { - "input": "What is the capital of France?" - }, - { - "input": "How do I make scrambled eggs?" - } - ], - "params": { - "parallelism": 4, - "limit_samples": 5, - "ignore_request_failure": false, - "request_timeout": 120, - "max_retries": 3 - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim" - }, - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "{{item.input}}" - } - ] - } -} \ No newline at end of file diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index d57986c00e..bade6c835d 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -177,6 +177,7 @@ async def compile( from nemo_evaluator.jobs.compiler import compile_evaluate_job canonical_spec = spec if isinstance(spec, EvaluateSpec) else EvaluateSpec.model_validate(spec.model_dump()) + canonical_spec.params = resolve_params(canonical_spec.params, canonical_spec.target) return compile_evaluate_job(canonical_spec, profile=profile) @staticmethod @@ -220,7 +221,7 @@ async def to_spec( else EvaluateInputSpec.model_validate(input_spec.model_dump()) ) metrics = [unbundle_metric(bundle) for bundle in submit_spec.metrics] - resolve_params(submit_spec.params, submit_spec.target) + submit_spec.params = resolve_params(submit_spec.params, submit_spec.target) unresolved_refs = _unresolved_model_refs(metrics) if unresolved_refs: if async_sdk is None: diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index 60a424c0f9..5023fa6b2d 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -10,9 +10,11 @@ from types import SimpleNamespace from typing import Any, Literal, cast +import nemo_evaluator.cli as evaluator_cli import pytest from nemo_evaluator.cli import EvaluatorPluginCLI from nemo_evaluator.filesets import FilesetRef +from nemo_evaluator.jobs.compiler import compile_evaluate_job from nemo_evaluator.jobs.evaluate import ( AGGREGATE_SCORES_RESULT_NAME, ARTIFACTS_RESULT_NAME, @@ -246,6 +248,32 @@ def _llm_judge_ref_metric() -> LLMJudgeMetric: ) +@pytest.mark.parametrize( + "spec_path", + [ + Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json"), + Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json"), + Path("skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json"), + ], +) +def test_example_spec_uses_metric_bundle_shape(spec_path: Path) -> None: + repo_root = Path(__file__).resolve().parents[3] + payload = json.loads((repo_root / spec_path).read_text(encoding="utf-8")) + + spec = EvaluateSpec.model_validate(payload) + compiled = compile_evaluate_job(spec) + + assert "metric" not in payload + assert len(spec.metrics) >= 1 + assert PlatformJobSpec.model_validate(compiled).steps[0].config is not None + for metric_payload in payload["metrics"]: + bundle = MetricBundle.model_validate(metric_payload) + # Static cloudpickle fixtures are Python-minor-version specific, so + # this test validates the checked-in bundle envelope without hydrating. + assert bundle.payload.kind == "cloudpickle" + assert bundle.metric_type == metric_payload["metric_type"] + + def test_evaluate_job_runs_inline_exact_match_metric() -> None: result = NemoJobScheduler().run_local(EvaluateJob, _exact_match_spec()) @@ -295,6 +323,88 @@ def test_cli_info_reports_registered_evaluator_job_key() -> None: assert payload["jobs"] == ["evaluator.evaluate"] +def test_cli_metric_types_reports_sdk_metric_union_types() -> None: + app = EvaluatorPluginCLI().get_cli() + + result = CliRunner().invoke(app, ["metric-types"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + entries = payload["metric_types"] + metric_names = [entry["name"] for entry in entries] + metrics = {entry["name"]: entry["description"] for entry in entries} + assert metrics["exact-match"].startswith("Exact-match metric runtime for evaluator-driven execution.") + assert metrics["llm-judge"].startswith("Runtime metric implementation for LLM-as-a-judge scoring.") + assert metrics["remote"].startswith("A metric that computes scores via a remote endpoint.") + assert metrics["topic_adherence"] == "Metric for measuring topic adherence." + assert "system" not in metrics + assert "system-retriever" not in metrics + + ragas_metric_types = { + "agent_goal_accuracy", + "answer_accuracy", + "context_entity_recall", + "context_precision", + "context_recall", + "context_relevance", + "faithfulness", + "noise_sensitivity", + "response_groundedness", + "response_relevancy", + "tool_call_accuracy", + "topic_adherence", + } + first_ragas_index = min(metric_names.index(metric_type) for metric_type in ragas_metric_types) + non_ragas_metric_types = metric_names[:first_ragas_index] + trailing_ragas_metric_types = metric_names[first_ragas_index:] + assert not ragas_metric_types.intersection(non_ragas_metric_types) + assert set(trailing_ragas_metric_types) == ragas_metric_types + assert non_ragas_metric_types == sorted(non_ragas_metric_types) + assert trailing_ragas_metric_types == sorted(trailing_ragas_metric_types) + + +def test_cli_metric_types_rejects_duplicate_metric_type_keys(mocker: MockerFixture) -> None: + class FirstMetric(BaseModel): + type: Literal["duplicate-metric"] = "duplicate-metric" + + class SecondMetric(BaseModel): + type: Literal["duplicate-metric"] = "duplicate-metric" + + mocker.patch.object( + evaluator_cli, + "_unwrap_metric_model_classes", + return_value=[FirstMetric, SecondMetric], + ) + + with pytest.raises( + ValueError, + match="Duplicate metric type 'duplicate-metric' mapped to both FirstMetric and SecondMetric", + ): + evaluator_cli._metric_type_models() + + +def test_cli_metric_types_reports_json_schema_for_named_metric_types() -> None: + app = EvaluatorPluginCLI().get_cli() + + result = CliRunner().invoke(app, ["metric-types", "exact-match"]) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["title"] == "ExactMatchMetric" + assert payload["properties"]["type"]["const"] == "exact-match" + assert "workspace" not in payload["properties"] + + +def test_cli_metric_types_rejects_unknown_metric_types_name() -> None: + app = EvaluatorPluginCLI().get_cli() + + result = CliRunner().invoke(app, ["metric-types", "missing-metric"]) + + assert result.exit_code != 0 + assert "Unknown metric name 'missing-metric'" in result.output + assert "nemo evaluator metric-types" in result.output + + def test_cli_run_executes_evaluator_job() -> None: app = EvaluatorPluginCLI().get_cli() add_job_commands(app, {"evaluator.evaluate": EvaluateJob}) @@ -519,6 +629,30 @@ async def test_evaluate_job_compile_produces_online_model_job() -> None: assert config["params"]["parallelism"] == 3 +async def test_evaluate_job_compile_normalizes_generic_online_model_params() -> None: + spec = EvaluateSpec.model_validate( + { + **_exact_match_spec(), + "target": Model(url="http://model.test/v1/chat/completions", name="test-model"), + "params": RunConfigOnline(parallelism=3), + "prompt_template": "Question: {{item.question}}", + } + ) + + compiled = await EvaluateJob.compile( + workspace="default", + spec=spec, + entity_client=object(), + job_name=None, + async_sdk=object(), + ) + + job_spec = PlatformJobSpec.model_validate(compiled) + config = cast(dict[str, Any], job_spec.steps[0].config) + assert isinstance(spec.params, RunConfigOnlineModel) + assert config["params"]["parallelism"] == 3 + + async def test_evaluate_job_compile_produces_online_agent_job() -> None: spec = EvaluateSpec.model_validate( { diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py index 6f29a88501..d3ed9f0de1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/execution/config.py @@ -24,6 +24,10 @@ def resolve_params( ) -> _RunConfigT: """Return params after validating that they match the selected target mode.""" if isinstance(target, Model): + if params is None or type(params) is RunConfig: + raise TypeError("model target requires RunConfigOnlineModel") + if type(params) is RunConfigOnline: + return RunConfigOnlineModel.model_validate(params.model_dump()) if not isinstance(params, RunConfigOnlineModel): raise TypeError("model target requires RunConfigOnlineModel") return params diff --git a/skills/nemo-evaluator-plugin/BENCHMARK.md b/skills/nemo-evaluator-plugin/BENCHMARK.md index 700fbd4d6d..78e3e7d918 100644 --- a/skills/nemo-evaluator-plugin/BENCHMARK.md +++ b/skills/nemo-evaluator-plugin/BENCHMARK.md @@ -7,7 +7,7 @@ This benchmark summarizes 3-Tier Evaluation from NVSkills-Eval results for the s ## Evaluation Summary - Skill: `nemo-evaluator-plugin` -- Evaluation date: 2026-05-30 +- Evaluation date: 2026-06-03 - NVSkills-Eval profile: `external` - Environment: `local` - Dataset: 1 evaluation tasks @@ -55,24 +55,24 @@ Task composition is derived from the evaluation dataset when possible. Entries w | Dimension | Num | `claude-code` | `codex` | |---|---:|---:|---:| | Security | 2 | 100% (+0%) | 100% (+0%) | -| Correctness | 2 | 92% (+8%) | 94% (+12%) | -| Discoverability | 2 | 61% (+27%) | 94% (+5%) | -| Effectiveness | 2 | 98% (+7%) | 92% (+29%) | -| Efficiency | 2 | 48% (+25%) | 92% (+7%) | +| Correctness | 2 | 92% (+0%) | 85% (+5%) | +| Discoverability | 2 | 63% (+0%) | 95% (+12%) | +| Effectiveness | 2 | 85% (-2%) | 70% (+8%) | +| Efficiency | 2 | 51% (+3%) | 93% (+15%) | Score values show skill-assisted performance. Values in parentheses show uplift versus the no-skill baseline when baseline data is available. ## Tier 1: Static Validation Summary -Tier 1 validation passed with observations. NVSkills-Eval ran 9 checks and found 10 total findings. +Tier 1 validation passed with observations. NVSkills-Eval ran 9 checks and found 12 total findings. Top findings: +- MEDIUM QUALITY/quality_correctness: No documented scripts in table format (`skills/nemo-evaluator-plugin/SKILL.md`) +- MEDIUM QUALITY/quality_correctness: Instructions don't mention 'run_script' (`skills/nemo-evaluator-plugin/SKILL.md`) - MEDIUM QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/nemo-evaluator-plugin/SKILL.md`) - MEDIUM QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/nemo-evaluator-plugin/SKILL.md`) - MEDIUM QUALITY/quality_efficiency: Deeply nested references in llm-judge.md (`skills/nemo-evaluator-plugin/SKILL.md`) -- MEDIUM SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/nemo-evaluator-plugin/SKILL.md`) -- MEDIUM SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nemo-evaluator-plugin/SKILL.md`) ## Tier 2: Deduplication Summary @@ -80,8 +80,8 @@ Tier 2 validation passed. NVSkills-Eval ran 2 checks and found 0 total findings. Notable observations: -- Context Deduplication: Collected 4 file(s) -- Inter-Skill Deduplication: Parsed skill 'nemo-evaluator-plugin': 103 char description +- Context Deduplication: Collected 6 file(s) +- Inter-Skill Deduplication: Parsed skill 'nemo-evaluator-plugin': 117 char description ## Publication Recommendation diff --git a/skills/nemo-evaluator-plugin/SKILL.md b/skills/nemo-evaluator-plugin/SKILL.md index e612a9783a..e9966e7927 100644 --- a/skills/nemo-evaluator-plugin/SKILL.md +++ b/skills/nemo-evaluator-plugin/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-evaluator-plugin -description: Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, or plugin-owned Evaluator skills. +description: Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, metric types, or plugin-owned Evaluator skills. metadata: owner: nemo-platform maturity: active @@ -9,18 +9,14 @@ license: Apache-2.0 # Evaluator Plugin -Use this skill when the task is about Evaluator functionality on the plugin architecture. The plugin-backed CLI surface is `nemo evaluator`; the legacy generated `nemo evaluation` API command group is not the target surface for new guidance. +Use this skill for evaluation tasks against a running NeMo Platform server. The plugin-backed CLI interface is `nemo evaluator`; the legacy generated `nemo evaluation` API command group is not the target surface for new guidance. -## Current Surfaces +## CLI Interface -- a minimal `nemo.services` health surface -- an SDK-backed `nemo.jobs` entry, `evaluator.evaluate`, for inline metric execution -- a minimal CLI and SDK namespace -- plugin-owned docs and skills directories +### Prerequisites -## CLI Commands - -> **Prerequisite:** activate the Python virtual environment before invoking the `nemo` CLI: `source .venv/bin/activate`. +- all commands in this file assume that the shell's working dir is at the root of the Nvidia-NeMo/nemo-platform repo +- activate the Python virtual environment before invoking the `nemo` CLI: `source .venv/bin/activate` Check plugin status from the CLI: @@ -28,69 +24,122 @@ Check plugin status from the CLI: nemo evaluator info ``` -Inspect the registered job contract: +## Metric Types + +### Explore Available Metrics + +To view available metric names, run: ```bash -nemo evaluator evaluate explain +nemo evaluator metric-types ``` -Run an inline `exact-match` metric: +To view a specific metric schema, pass a metric name from the `metric_types` list above: ```bash -nemo evaluator evaluate run --spec '{"metric":{"type":"exact-match","reference":"{{item.expected}}","candidate":"{{item.model_output}}"},"dataset":[{"expected":"blue","model_output":"Blue"},{"expected":"Jupiter","model_output":"Saturn"}],"params":{"parallelism":2}}' +nemo evaluator metric-types ``` -Run an inline `string-check` metric: +Inspect all the registered metric schema contracts: ```bash -nemo evaluator evaluate run --spec '{"metric":{"type":"string-check","operation":"contains","left_template":"{{item.answer}}","right_template":"NeMo"},"dataset":[{"answer":"NeMo Platform supports evaluator plugins."}]}' +nemo evaluator evaluate explain ``` -For non-trivial specs, prefer `--spec-file` over inline shell JSON: +> Note: use `nemo evaluator evaluate explain` as the source of truth for the current plugin input schema. It will return a large json schema response, so strongly prefer `nemo evaluator metric-types` when you only need metric names and corresponding schemas. + +## Evaluation Spec + +Evaluation spec is a payload that is provided to CLI as an input to execute evaluation. + +At a high level, a spec describes: + +- `metrics`: bundled Evaluator SDK metric configurations +- `dataset`: inline rows to evaluate or platform FilesetRef that contains the dataset +- `params`: optional Evaluator SDK execution parameters +- `target`: optional model or agent target for online evaluation + +See the LLM-judge spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json). + +### Metric Bundle Payloads + +The checked-in [spec examples](./assets/specs) use bundled SDK metrics. The fields under `metrics[*].payload` are generated by `bundle_metric(metric, CloudpickleMetricBundlePackager())`. + +To see the pattern for configuring a pre-defined SDK metric, for example `ExactMatchMetric`, and converting it into bundled metric JSON, inspect `build_metric_bundle_example()` in [generate_example_specs.py](./scripts/generate_example_specs.py) and run: ```bash -nemo evaluator evaluate run --spec-file evaluation-spec.json +uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py ``` -Submit the same spec to a cluster: +## Run Evaluations + +### Run Using File Spec Reference + +When using the `nemo evaluator evaluate run` command, results are saved into local temporary directories and the link is printed to stdout. +Prefer the `--spec-file` named argument over inline shell JSON because metric bundles include serialized payloads. +Examples of various specs are provided in the [assets/specs](./assets/specs/) directory. + +#### Evaluate using `exact-match` metric + +See the spec example at [assets/specs/exact_match_metric.json](./assets/specs/exact_match_metric.json). ```bash -nemo evaluator evaluate submit \ - --spec-file evaluation-spec.json \ - --workspace default \ - --profile default +nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +``` + +#### Evaluate using a benchmark metric set + +```bash +nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json ``` -Use `nemo evaluator evaluate explain` as the source of truth for the current plugin job schema. +#### Evaluate using `LLM-Judge` metric -## Evaluation Specs +Uses an LLM to score responses. See the spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json). -The current job accepts inline SDK-backed evaluation specs. At a high level, specs describe: +```bash +nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json +``` -- `metric`: inline Evaluator SDK metric configuration or benchmark metrics -- `dataset`: inline rows to evaluate -- `params`: optional Evaluator SDK execution parameters -- `target`: optional model or agent target for online evaluation +### Run Evaluation As A Durable Job -For LLM-judge setup notes, see [LLM Judge Notes](references/llm-judge.md). +Use the `nemo evaluator evaluate submit` command to create a durable evaluation job. The response of this command returns a job handler object instead of the evaluation result. -For evaluator API key auth, see [Evaluator API Auth](references/api-auth.md). +```bash +nemo evaluator evaluate submit \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +``` -For local and cluster troubleshooting, see [Evaluation Troubleshooting](references/troubleshooting.md). +The submit response includes the generated job's `name` field, for example `nemo-evaluator-zlhn1ecd`. Wait for the job to complete, then list and download the job results. + +```bash +nemo jobs get-status +nemo jobs get +nemo jobs results list +nemo jobs results download aggregate-scores --job --output-file aggregate-scores.json +nemo jobs results download row-scores --job --output-file row-scores.jsonl +``` + +## Python SDK Interface -Call the SDK-backed status route through the platform SDK: +Evaluator Python SDK client is exposed as `evaluator` variable on `NeMoPlatform` instance: ```python from nemo_platform import NeMoPlatform -client = NeMoPlatform(base_url="http://localhost:8000") -status = client.evaluator.plugin_status() +platform_client = NeMoPlatform(base_url="http://localhost:8080") +status = platform_client.evaluator.plugin_status() ``` -## Next Decisions +See examples of using the plugin SDK interface in [plugin_sdk_examples.py](./assets/examples/plugin_sdk_examples.py). + +## Security +Make sure not to print any secrets to stdout since this can be collected as logs -Before replacing stubs, verify the target surface: +## Additional Resources -1. service route adaptation -2. job submission or compilation strategy -3. packaging split between service and task dependencies +For LLM-judge setup notes, see [LLM Judge Notes](references/llm-judge.md). + +For evaluator API key auth, see [Evaluator API Auth](references/api-auth.md). + +For local and cluster troubleshooting, see [Evaluation Troubleshooting](references/troubleshooting.md). diff --git a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py new file mode 100644 index 0000000000..e572ae5bb6 --- /dev/null +++ b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Local-only Evaluator plugin SDK smoke example. + +The default entrypoint prints an exact-match spec and does not submit jobs or +call hosted models. Pass --run to execute the same offline metric against a +running local NeMo Platform. +""" + +from __future__ import annotations + +import argparse +import gzip +import json +import os +from collections.abc import Iterable +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +DEFAULT_BASE_URL = "http://localhost:8080" +DEFAULT_ROWS = ( + {"expected": "blue", "model_output": "blue"}, + {"expected": "Jupiter", "model_output": "Saturn"}, +) + + +def write_jsonl_dataset(path: Path, rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> Path: + """Write rows as JSONL and return the written path.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return path + + +def load_jsonl_rows(path: Path, *, limit: int | None = None) -> list[dict[str, Any]]: + """Load plain JSONL or .gz JSONL rows.""" + opener = gzip.open if path.suffix == ".gz" else open + rows: list[dict[str, Any]] = [] + + with opener(path, "rt", encoding="utf-8") as stream: + for line in stream: + if line.strip(): + rows.append(json.loads(line)) + if limit is not None and len(rows) >= limit: + break + + return rows + + +def build_exact_match_spec(rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> dict[str, Any]: + """Build a local exact-match spec that does not require model credentials.""" + from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric + from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric + + metric = ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.model_output}}", + ) + return { + "metrics": [bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json")], + "dataset": list(rows), + "params": {"parallelism": 2, "limit_samples": 2}, + } + + +def run_local_exact_match(dataset_path: Path) -> Any: + """Run the offline exact-match metric against a local platform.""" + from nemo_evaluator.sdk.types import RunConfig + from nemo_evaluator_sdk.enums import MetricType + from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric + from nemo_platform import NeMoPlatform + + client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", DEFAULT_BASE_URL), + workspace="default", + ) + try: + evaluator = client.evaluator + metric = ExactMatchMetric( + type=MetricType.EXACT_MATCH, + reference="{{item.expected}}", + candidate="{{item.model_output}}", + ) + return evaluator.run(metric=metric, dataset=dataset_path, config=RunConfig(limit_samples=2)) + finally: + client.close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", action="store_true", help="Run local offline exact-match against NeMo Platform.") + args = parser.parse_args(argv) + + with TemporaryDirectory(prefix="nemo-evaluator-smoke-") as tmpdir: + dataset_path = write_jsonl_dataset(Path(tmpdir) / "exact-match.jsonl") + + if args.run: + result = run_local_exact_match(dataset_path) + result.print_summary() + return 0 + + print(json.dumps(build_exact_match_spec(load_jsonl_rows(dataset_path)), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json new file mode 100644 index 0000000000..6cf51562c0 --- /dev/null +++ b/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json @@ -0,0 +1,96 @@ +{ + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "exact-match", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "exact-match", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "python_version": "3.11.15", + "cloudpickle_version": "3.1.2", + "pickle_protocol": 5, + "blob": "gAWVoQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwSe3tpdGVtLnJlZmVyZW5jZX19lIwJY2FuZGlkYXRllE51jBJfX3B5ZGFudGljX2V4dHJhX1-UTowXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QoaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", + "digest": "b6d94b6d5a4f304964652358cd55e8e1216664934a9712ac147d566b65ed3b5d", + "kind": "cloudpickle" + } + }, + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "string-check", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "string-check", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "python_version": "3.11.15", + "cloudpickle_version": "3.1.2", + "pickle_protocol": 5, + "blob": "gAWV5gEAAAAAAACMJ25lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLnN0cmluZ19jaGVja5SMEVN0cmluZ0NoZWNrTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAxzdHJpbmctY2hlY2uUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJb3BlcmF0aW9ulIwIY29udGFpbnOUjA1sZWZ0X3RlbXBsYXRllIwWe3tzYW1wbGUub3V0cHV0X3RleHR9fZSMDnJpZ2h0X3RlbXBsYXRllIwYe3tpdGVtLnJlcXVpcmVkX3BocmFzZX19lHWMEl9fcHlkYW50aWNfZXh0cmFfX5ROjBdfX3B5ZGFudGljX2ZpZWxkc19zZXRfX5SPlChoHmggaAdoHJCMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", + "digest": "5c7c5b74de79b3d84fdd7db4f52393633dee4370036266d088518b94a92b081b", + "kind": "cloudpickle" + } + } + ], + "dataset": [ + { + "prompt": "Return exactly this word with no punctuation: Paris", + "reference": "Paris", + "required_phrase": "Paris" + }, + { + "note": "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are 'London' so both metrics should report a miss.", + "prompt": "Return exactly this word with no punctuation: Oslo", + "reference": "London", + "required_phrase": "London" + } + ], + "params": { + "parallelism": 4, + "limit_samples": 2, + "ignore_request_failure": false, + "request_timeout": 60, + "max_retries": 3 + }, + "target": { + "url": "https://integrate.api.nvidia.com/v1/chat/completions", + "name": "nvidia/nemotron-3-super-120b-a12b", + "api_key_secret": "NVIDIA_API_KEY", + "format": "nim" + }, + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "{{item.prompt}}" + } + ] + } +} diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json new file mode 100644 index 0000000000..4eaee1a54b --- /dev/null +++ b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json @@ -0,0 +1,46 @@ +{ + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "exact-match", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "exact-match", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "python_version": "3.11.15", + "cloudpickle_version": "3.1.2", + "pickle_protocol": 5, + "blob": "gAWVuQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwRe3tpdGVtLmV4cGVjdGVkfX2UjAljYW5kaWRhdGWUjBV7e2l0ZW0ubW9kZWxfb3V0cHV0fX2UdYwSX19weWRhbnRpY19leHRyYV9flE6MF19fcHlkYW50aWNfZmllbGRzX3NldF9flI-UKGgeaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", + "digest": "38050e2438a5eef8865ee2ef0bc2ccdaff6991b5d91bd9fb94a8155e759a26a5", + "kind": "cloudpickle" + } + } + ], + "dataset": [ + { + "expected": "blue", + "model_output": "Blue" + }, + { + "expected": "Jupiter", + "model_output": "Saturn" + } + ], + "params": { + "parallelism": 2 + } +} diff --git a/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json new file mode 100644 index 0000000000..092072c428 --- /dev/null +++ b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json @@ -0,0 +1,63 @@ +{ + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "helpfulness", + "description": "How well does the response help the user?", + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": { + "NVIDIA_API_KEY": "NVIDIA_API_KEY" + }, + "payload": { + "python_version": "3.11.15", + "cloudpickle_version": "3.1.2", + "pickle_protocol": 5, + "blob": "gAWVOAkAAAAAAACMJG5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmxsbV9qdWRnZZSMDkxMTUp1ZGdlTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAlsbG0tanVkZ2WUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwFbW9kZWyUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLm1vZGVsc5SMBU1vZGVslJOUKYGUfZQoaAV9lCiMA3VybJSMNGh0dHBzOi8vaW50ZWdyYXRlLmFwaS5udmlkaWEuY29tL3YxL2NoYXQvY29tcGxldGlvbnOUjARuYW1llIwhbnZpZGlhL25lbW90cm9uLTMtc3VwZXItMTIwYi1hMTJilIwPZGVmYXVsdF9oZWFkZXJzlE6MCGhvc3RfdXJslE6MDmFwaV9rZXlfc2VjcmV0lGgTjAlTZWNyZXRSZWaUk5QpgZR9lChoBX2UjARyb290lIwOTlZJRElBX0FQSV9LRVmUc4wXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QojARyb290lJB1YowGZm9ybWF0lGgIjAtNb2RlbEZvcm1hdJSTlIwDbmltlIWUUpR1jBJfX3B5ZGFudGljX2V4dHJhX1-UTmgxj5QoaCloI2glaDSQjBRfX3B5ZGFudGljX3ByaXZhdGVfX5ROdWKMBnNjb3Jlc5RdlIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5zY29yZXOUjApSYW5nZVNjb3JllJOUKYGUfZQoaAV9lChoJYwLaGVscGZ1bG5lc3OUaA6MKUhvdyB3ZWxsIGRvZXMgdGhlIHJlc3BvbnNlIGhlbHAgdGhlIHVzZXI_lIwGcGFyc2VylGg_jA9KU09OU2NvcmVQYXJzZXKUk5QpgZR9lChoBX2UKGgHjARqc29ulIwJanNvbl9wYXRolGhFdWg6Tmgxj5QoaE6QaDxOdWKMB21pbmltdW2USwCMB21heGltdW2USwR1aDpOaDGPlChoR2hRaA5oJWhQkGg8TnViYYwPcHJvbXB0X3RlbXBsYXRllH2UjAhtZXNzYWdlc5RdlCh9lCiMBHJvbGWUjAZzeXN0ZW2UjAdjb250ZW50lIyGWW91IGFyZSBhbiBldmFsdWF0b3IuIFJhdGUgdGhlIHJlc3BvbnNlJ3MgaGVscGZ1bG5lc3MgZnJvbSAwLTQuIFJldHVybiBvbmx5IGEgSlNPTiBvYmplY3Qgd2l0aCB0aGlzIHNoYXBlOiB7ImhlbHBmdWxuZXNzIjogPGludGVnZXI-fS6UdX2UKGhYjAR1c2VylGhajHNVc2VyIHByb21wdDoge3tpdGVtLmlucHV0fX0KCkFzc2lzdGFudCByZXNwb25zZToge3tzYW1wbGUub3V0cHV0X3RleHQgfCBkZWZhdWx0KGl0ZW0ub3V0cHV0KX19CgpSYXRlIHRoaXMgcmVzcG9uc2UulHVlc4wPb3B0aW9uYWxfZmllbGRzlF2UjBFzdHJ1Y3R1cmVkX291dHB1dJR9lIwGc2NoZW1hlH2UKGgHjAZvYmplY3SUjApwcm9wZXJ0aWVzlH2UaEV9lChoB4wHaW50ZWdlcpRoUEsAaFFLBHVzjAhyZXF1aXJlZJRdlGhFYXVzjAlpbmZlcmVuY2WUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLnBhcmFtc5SMD0luZmVyZW5jZVBhcmFtc5STlCmBlH2UKGgFfZQojAt0ZW1wZXJhdHVyZZRHAAAAAAAAAACMCm1heF90b2tlbnOUTQCAjBVtYXhfY29tcGxldGlvbl90b2tlbnOUTowFdG9wX3CUTowEc3RvcJROdWg6fZRoMY-UKGhzaHSQaDxOdWKMDXN5c3RlbV9wcm9tcHSUTowJcmVhc29uaW5nlE6MFmlnbm9yZV9yZXF1ZXN0X2ZhaWx1cmWUiYwIam9iX3R5cGWUaBh1aDpOaDGPlChoX2gcaFNobGg9aGFofJBoPH2UKIwRX3ByZXByb2Nlc3NfaG9va3OUXZQojBxuZW1vX2V2YWx1YXRvcl9zZGsuaW5mZXJlbmNllIwVQWRkSW5mZXJlbmNlUGFyYW1ldGVylJOUKYGUfZSMBnBhcmFtc5R9lChoc0cAAAAAAAAAAGh0TQCAdXNijCRuZW1vX2V2YWx1YXRvcl9zZGsuc3RydWN0dXJlZF9vdXRwdXSUjBlJbmZlcmVuY2VTdHJ1Y3R1cmVkT3V0cHV0lJOUKYGUfZQojAxfanNvbl9zY2hlbWGUfZQoaAdoZWhmaGdoamhrdYwHX3N0cmljdJSJjARtb2RllGiJjBRTdHJ1Y3R1cmVkT3V0cHV0TW9kZZSTlIwRbnZleHRfZ3VpZGVkX2pzb26UhZRSlIwPaW5mZXJlbmNlX3BhcmFtlH2UjApleHRyYV9ib2R5lH2UjAVudmV4dJR9lIwLZ3VpZGVkX2pzb26UaI9zc3N1YmiCjAdMb2dIb29rlJOUKYGUfZSMBmxvZ2dlcpSMB2xvZ2dpbmeUjAlnZXRMb2dnZXKUk5RogoWUUpRzYmWMEl9wb3N0cHJvY2Vzc19ob29rc5RdlGigYYwaX3VzZV9tYXhfY29tcGxldGlvbl90b2tlbnOUiYwIX2FwaV9rZXmUTowHX2NsaWVudJROjA1faW5mZXJlbmNlX2ZulE6MCF9wYXJzZXJzlH2UaEVoP4wPU2NvcmVQYXJzZXJKU09OlJOUKYGUfZQojAVzY29yZZRoQmhOaEVoYWhijAtqc29uX3NjaGVtYZRoZHVic4wMX3Njb3JlX2R1bXBzlH2UaEV9lChoJWhFaA5oRmhQSwBoUUsEdXOMG19wcm9tcHRfdGVtcGxhdGVfaXNfZGVmYXVsdJSJdXViLg==", + "digest": "dfd6a04359b75b41cba2817bc1496244425e8290102b36c20bd04e7f62b31b8e", + "kind": "cloudpickle" + } + } + ], + "dataset": [ + { + "input": "What is the capital of France?" + }, + { + "input": "How do I make scrambled eggs?" + } + ], + "params": { + "parallelism": 2, + "limit_samples": 2, + "request_timeout": 120, + "max_retries": 3 + }, + "target": { + "url": "https://integrate.api.nvidia.com/v1/chat/completions", + "name": "nvidia/nemotron-3-super-120b-a12b", + "api_key_secret": "NVIDIA_API_KEY", + "format": "nim" + }, + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "{{item.input}}" + } + ] + } +} diff --git a/skills/nemo-evaluator-plugin/references/api-auth.md b/skills/nemo-evaluator-plugin/references/api-auth.md index 56182dc2a9..4c69361724 100644 --- a/skills/nemo-evaluator-plugin/references/api-auth.md +++ b/skills/nemo-evaluator-plugin/references/api-auth.md @@ -3,7 +3,7 @@ Use the correct `model.api_key_secret` (if `model` is used) for the evaluator execution mode: - Local `nemo evaluator evaluate run`: `api_key_secret` is the name of an environment variable available to the local process, such as `NVIDIA_API_KEY`. -- Remote `nemo evaluator evaluate submit`: `api_key_secret` is the name of a NeMo platform secret in the target workspace. +- Remote `nemo evaluator evaluate submit`: `api_key_secret` is the name of a NeMo platform secret in the target workspace, such as `nvidia-api-key`. The remote job runtime cannot read local environment variables. In remote mode, if a model sets `api_key_secret`, create or verify the platform secret before submitting the job: @@ -11,3 +11,5 @@ The remote job runtime cannot read local environment variables. In remote mode, printf '%s' "$NVIDIA_API_KEY" | nemo secrets create nvidia-api-key --from-file - nemo secrets list ``` + +If you copy a local LLM-judge spec that uses `"api_key_secret": "NVIDIA_API_KEY"` for remote submission, change that value to the platform secret name, for example `"nvidia-api-key"`. diff --git a/skills/nemo-evaluator-plugin/references/llm-judge.md b/skills/nemo-evaluator-plugin/references/llm-judge.md index f909f6a07e..2052924ca1 100644 --- a/skills/nemo-evaluator-plugin/references/llm-judge.md +++ b/skills/nemo-evaluator-plugin/references/llm-judge.md @@ -16,7 +16,9 @@ For local iteration, keep the metric and dataset in a spec file and run: nemo evaluator evaluate run --spec-file evaluation-spec.json ``` -For cluster execution, submit the same spec: +The checked-in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json` is a local-run example. It expects `NVIDIA_API_KEY` to be set in the local shell. + +For durable execution, submit the same spec: ```bash nemo evaluator evaluate submit \ @@ -25,4 +27,6 @@ nemo evaluator evaluate submit \ --profile default ``` +Before submitting an LLM-judge spec via `submit`, replace local environment-variable names with platform secret names, such as `nvidia-api-key`. + Prefer `--spec-file` over inline `--spec` for LLM-judge metrics because prompts and score definitions quickly become hard to audit as shell-escaped JSON. diff --git a/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py new file mode 100644 index 0000000000..953454e8e3 --- /dev/null +++ b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Print an exact-match metric bundle example. + +Run from the repo root: + + uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py +""" + +from __future__ import annotations + +import json +import os +import sys +from typing import Any + +DETERMINISTIC_HASH_SEED = "0" +JSON_OUTPUT_INDENT = 4 +SUCCESS_EXIT_CODE = 0 + + +def _ensure_deterministic_hash_seed() -> None: + if os.environ.get("PYTHONHASHSEED") == DETERMINISTIC_HASH_SEED: + return + env = {**os.environ, "PYTHONHASHSEED": DETERMINISTIC_HASH_SEED} + os.execvpe(sys.executable, [sys.executable, *sys.argv], env) + + +def _bundle(metric: Any) -> dict[str, Any]: + _ensure_deterministic_hash_seed() + + from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric + from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager + + return bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json") + + +def build_metric_bundle_example() -> dict[str, Any]: + """Return bundled JSON for one configured SDK metric.""" + from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric + + metric = ExactMatchMetric( + reference="{{item.gold_answer}}", + candidate="{{item.prediction}}", + ) + return _bundle(metric) + + +def main() -> int: + print(json.dumps(build_metric_bundle_example(), indent=JSON_OUTPUT_INDENT)) + return SUCCESS_EXIT_CODE + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/nemo-evaluator-plugin/skill-card.md b/skills/nemo-evaluator-plugin/skill-card.md index 6fbeabdd43..421453fff9 100644 --- a/skills/nemo-evaluator-plugin/skill-card.md +++ b/skills/nemo-evaluator-plugin/skill-card.md @@ -1,5 +1,5 @@ ## Description:
-Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, or plugin-owned Evaluator skills.
+Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, metric types, or plugin-owned Evaluator skills.
This skill is ready for commercial/non-commercial use.
@@ -7,9 +7,9 @@ This skill is ready for commercial/non-commercial use.
NVIDIA
### License/Terms of Use:
-Apache-2.0
+Apache 2.0
## Use Case:
-Developers and engineers use this skill to configure, run, and troubleshoot NeMo Evaluator plugin jobs for inline metric execution and benchmark evaluation of models and agents.
+Developers and engineers who need to run evaluation tasks (exact-match metrics, LLM-as-judge scoring, benchmark suites, and durable evaluation jobs) against a running NeMo Platform server.
### Deployment Geography for Use:
Global
@@ -19,14 +19,16 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Evaluator API Auth](references/api-auth.md)
- [LLM Judge Notes](references/llm-judge.md)
+- [Evaluator API Auth](references/api-auth.md)
- [Evaluation Troubleshooting](references/troubleshooting.md)
+- [NeMo Platform Documentation](https://nvidia-nemo.github.io/nemo-platform/)
+- [Berkeley Function Calling Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html)
## Skill Output:
-**Output Type(s):** [Shell commands, Code, Configuration instructions]
-**Output Format:** [Markdown with inline bash and Python code blocks]
+**Output Type(s):** [Shell commands, API Calls, JSON, Configuration instructions]
+**Output Format:** [Markdown with inline bash code blocks and JSON spec files]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -37,7 +39,7 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-Evaluated against 1 internal skill evaluation task with 2 attempts per task (pass threshold: 50%).
+Evaluated against 1 evaluation task (positive skill-activation case) with 2 attempts per task via NVSkills-Eval external profile.
## Evaluation Metrics Used:
Reported benchmark dimensions:
@@ -62,10 +64,10 @@ Underlying evaluation signals used in this run:
| Dimension | Num | `claude-code` | `codex` | |---|---:|---:|---:| | Security | 2 | 100% (+0%) | 100% (+0%) | -| Correctness | 2 | 92% (+8%) | 94% (+12%) | -| Discoverability | 2 | 61% (+27%) | 94% (+5%) | -| Effectiveness | 2 | 98% (+7%) | 92% (+29%) | -| Efficiency | 2 | 48% (+25%) | 92% (+7%) | +| Correctness | 2 | 92% (+0%) | 85% (+5%) | +| Discoverability | 2 | 63% (+0%) | 95% (+12%) | +| Effectiveness | 2 | 85% (-2%) | 70% (+8%) | +| Efficiency | 2 | 51% (+3%) | 93% (+15%) | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/nemo-evaluator-plugin/skill.oms.sig b/skills/nemo-evaluator-plugin/skill.oms.sig index ff9b99e031..45a1d2311e 100644 --- a/skills/nemo-evaluator-plugin/skill.oms.sig +++ b/skills/nemo-evaluator-plugin/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1ldmFsdWF0b3ItcGx1Z2luIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImY2YzE4MjQyYmIxNzgyNGM3OWJhZDgxMjRmNWNiYzE0NmEyYjM0YzMyZDAzYjgyYTAyMWM0ODkzMmYxY2M3ZmIiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdCIKICAgICAgXSwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI1NjQ1NTRhY2YwNzY3YTA5N2NjY2EzZDQ5YTc0MDRhOTRhZTgyNjgzYWYyYmFjNDFmMGFmMjQ1MjFlYmFlOGRmIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkNTJhNjVmMTRhMjdlNDRlZDM4MWM4NWRmMGUwNTFlZWNkODJlYzZlYmFjOTljNDU3MjJmNGMwNjI3MGE1MGYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIzYWZiMzA3ODY1MDUzMTJlYjUzNmQ5N2VmYzc1ZjUzODlhOWQ1Mjc0ZmEzMzk1ZTE0MDVmZDczMWRhYTcwM2IyIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcGktYXV0aC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZGM1YzJmZmI0ZTEzMjNlZTM0MDMyYmZiN2E2MzM0ZTk1MWVjZTg1OGE4M2RlOTJmMzYxZjBhM2M3ODM5OTc3NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvbGxtLWp1ZGdlLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwNmIyZjA3ZmFmM2Q5OTc1NmFhZGQ4NTRlOTY1YTUwNTExZTE2M2FiMmM2NGVkM2YzNmRjZDQ4Y2I2MzllMWVkIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy90cm91Ymxlc2hvb3RpbmcubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImE1YzI2MjEzZTkwODQxOGYxNjYyZmQ2MGRmNTg5MWRhNTcxNzVmMmEzZWM0ODhmNTc1N2Y3MDI5ZmU4MWFlZGEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI4ZDYwMjM3NmUxYWJkY2EzZDFkZDQxM2RjZmU2YmVjNGQ0ZTE5NDYyZjc3MGNkMjc5YmJmYTVmYTkzZTQ1ZTk2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMHUq1xhSGzmiB7eZJDroVKCTFHusQP1mKJHWl6N+GV/YAK5tdTmhkXtoM4RcSrZsugIxAPn0tnp6+HRn/craJaaOUgKLjvfHYTstuuxmFXHdlnfk2c2fm3sO2SmfbZN6PxS7mw==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1ldmFsdWF0b3ItcGx1Z2luIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogImE1YjdiMTQ5OGIxMzk3YTJlZjNmMmQwNmVmM2JiNDI1NTczZTZkNmExZGRiNzg3MGE1MTdiNjA4MDk1MDllNGQiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIgogICAgICBdLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIyMGViYTk5NGJlZDA3MjlhMDg3YjM4Y2E3ZWEwZDIwYWI5ZDAyZGVhYjdmZjFmYzdhYTQ3OGFhNWUzMjQzZTQ3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3ZjgzYTQxNmExMjUyYzc4ZjYwYTdlZjNkZjU3ODBiZWFmZWYyYTcwNjllMDM3ZjQwOGZmYmYzZjI3YjI3ZDI2IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImYzNzg2YTU3MjcwNjI1N2M1NGViYzJlY2E0ZmRiMmNlYTMxYmE1N2QzOWNjYzAyNGQ2MTE1OTZjNWVlNDc4NDIiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJhc3NldHMvZXhhbXBsZXMvcGx1Z2luX3Nka19leGFtcGxlcy5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNjJiMzNkZWE3NjJlYTc1MDUyZWM1MDU4ZjhlYmU1MDcyZGIxMGM3YTA3ZDlmNjY2ZTBkNmEzY2Q5NzM5NmJjNyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFzc2V0cy9zcGVjcy9leGFjdF9tYXRjaF9iZW5jaG1hcmsuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZjBjOWQ3YjFlMzNkZTExYmZjNTMxMDgyNzYxOWM5OGNmNGQ4NTgzMDE1MTFlNmRjMjI0Mzk0MGU2Y2ZkOTZiZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFzc2V0cy9zcGVjcy9leGFjdF9tYXRjaF9tZXRyaWMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNDg5NTZhZjJhMzRlNDJiODBiMDlmNjc4NjRkYzVlNjI5NGJlNWZhNTRjYmJmYjhiNjg3Y2JmZDEyNTA1ZjRkYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImFzc2V0cy9zcGVjcy9sbG1fYXNfanVkZ2UuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiM2FmYjMwNzg2NTA1MzEyZWI1MzZkOTdlZmM3NWY1Mzg5YTlkNTI3NGZhMzM5NWUxNDA1ZmQ3MzFkYWE3MDNiMiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImEwZWE0YTZmNzA4YWVhNGMwYzE0YmE2YzVkYmU1NTU1NjJlYWExYzJkZmZlYTFlZTlhN2IyZjE4ZmQ2YzVlMzgiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FwaS1hdXRoLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzN2IxMTExOWM1ZmIyY2FjYWY5NzAyM2NiMDViY2RlNGViY2NjNjIxMjhlMTJjNTM1ZWY4MzI1ZDNlZjYxMmJiIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9sbG0tanVkZ2UubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImE1YzI2MjEzZTkwODQxOGYxNjYyZmQ2MGRmNTg5MWRhNTcxNzVmMmEzZWM0ODhmNTc1N2Y3MDI5ZmU4MWFlZGEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3Ryb3VibGVzaG9vdGluZy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMzE3YjQzN2ViOTk5MjJhOTMwOTBkZDY3NzY4ODc5Njc3NzRhZDJjZTVkMmIyZTVhMTNjZDg2Zjk2ZWE1M2Q0ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvZ2VuZXJhdGVfZXhhbXBsZV9zcGVjcy5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNTQ5MGQyZThkMTgyMDZhMjZmOTY0MTdmNDRiNTQ2MDE5NmRkYTE2MzEyZmRlMDQxYThjMjM1MDkyZmJhMWMwYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCtGjq3A/tQBLjG9mJMnfZx+r+Ce6qv7GF7NjqeOrSkvmZ1r0oAWeV9T+schu5MmQoCMF0vvbcNDo8PHkV641V7P/85hcM4yBT3z048d2bQVyI9sZah017w98brHlCmMHLy3g==","keyid":""}]}} \ No newline at end of file diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/generator.py b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/generator.py index 680ef52cbb..f40490afef 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/generator.py +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/generator.py @@ -20,7 +20,14 @@ from nemo_platform_sdk_tools.license.format_osv_licenses import format_licenses_table from nemo_platform_sdk_tools.license.formats import get_formatter -from nemo_platform_sdk_tools.license.license_utils import ALLOWED_LICENSES, get_local_packages, resolve_license +from nemo_platform_sdk_tools.license.license_utils import ( + ALLOWED_LICENSES, + get_local_packages, + get_override_key_for_package, + normalize_package_name, + resolve_license, +) +from packaging.requirements import InvalidRequirement, Requirement logger = logging.getLogger(__name__) @@ -153,6 +160,48 @@ def run_osv_scanner(lockfile: Path, output_file: Path, cwd: Optional[Path] = Non raise LicenseGenerationError("osv-scanner command not found") +def _get_requirements_with_overrides( + requirements_file: Path, overrides: dict[str, str], local_packages: set[str] +) -> list[dict[str, str]]: + """Return exported requirements that can be licensed from reviewed overrides.""" + if not requirements_file.exists(): + return [] + + packages = [] + with open(requirements_file, encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if not stripped or stripped.startswith("#") or line.startswith((" ", "\t")): + continue + if stripped == "-e" or stripped.startswith("-e "): + continue + + requirement_text = stripped.removesuffix("\\").strip() + try: + requirement = Requirement(requirement_text) + except InvalidRequirement: + logger.warning("Could not parse exported requirement: %s", requirement_text) + continue + + name = requirement.name + if normalize_package_name(name) in local_packages: + continue + + version = "" + for specifier in requirement.specifier: + if specifier.operator == "==": + version = specifier.version + break + + override_key = get_override_key_for_package(name, version) + if override_key not in overrides: + continue + + packages.append({"name": name, "version": version, "license": overrides[override_key].upper()}) + + return packages + + def format_licenses( osv_json: Path, output_file: Path, overrides_file: Optional[Path] = None, format_type: str = "table" ) -> None: @@ -208,11 +257,6 @@ def format_licenses( licenses = pkg_data.get("licenses", []) # Skip local packages - from nemo_platform_sdk_tools.license.license_utils import ( - get_override_key_for_package, - normalize_package_name, - ) - if normalize_package_name(name) in local_packages: continue @@ -228,6 +272,12 @@ def format_licenses( packages.append({"name": name, "version": version, "license": license_str.upper()}) + # OSV can emit a partial package list when the service is degraded. + # Keep output stable for reviewed licenses by filling only packages + # that are present in the exported requirements and overrides.yaml. + requirements_file = osv_json.parent / "requirements-main.txt" + packages.extend(_get_requirements_with_overrides(requirements_file, overrides, local_packages)) + # Deduplicate by name (keep first occurrence) seen_names = set() unique_packages = [] diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml index de7de42063..9138253dbd 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml @@ -144,6 +144,7 @@ overrides: xxhash: BSD-3-Clause # xxHash hashing algorithm # Async / Concurrency + cloudpickle: BSD-3-Clause # https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE nest-asyncio: BSD-3-Clause # Patch asyncio to allow nested event loops nest-asyncio2: BSD-3-Clause # Forked nest-asyncio used by nvidia-nat-core diff --git a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py index 1431b686bf..f7fe6dca6d 100644 --- a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py +++ b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py @@ -3,6 +3,9 @@ """Tests for license utility functions.""" +import json +from typing import Any, cast + import pytest from nemo_platform_sdk_tools.license.license_utils import ( get_override_key_for_package, @@ -118,10 +121,10 @@ def test_real_world_ormsgpack_case(self): def test_invalid_type_raises_error(self): """Test that invalid input types raise appropriate errors.""" with pytest.raises(TypeError, match="licenses must be a list or str"): - resolve_license(123) # type: ignore[arg-type] + resolve_license(cast(Any, 123)) with pytest.raises(TypeError, match="licenses must be a list or str"): - resolve_license(None) # type: ignore[arg-type] + resolve_license(cast(Any, None)) def test_single_license_list(self): """Test list with single license.""" @@ -195,3 +198,30 @@ def test_format_licenses_table_applies_override_for_cu129_version(self): assert "torchao" in result # Override was applied, so we should not see UNKNOWN for this package assert "✘" not in result or "BSD-3-CLAUSE" in result + + +class TestFormatLicenses: + """Tests for license report formatting.""" + + def test_format_licenses_fills_missing_osv_package_from_overrides(self, tmp_path): + """A reviewed override fills an exported requirement omitted by OSV.""" + from nemo_platform_sdk_tools.license.generator import format_licenses + + license_dir = tmp_path / "third_party" + license_dir.mkdir() + osv_json = license_dir / "osv-licenses.json" + osv_json.write_text(json.dumps({"results": [{"packages": []}]}), encoding="utf-8") + (license_dir / "requirements-main.txt").write_text( + "cloudpickle==3.1.2 ; python_version >= '3.11' \\\n" + " --hash=sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414\n", + encoding="utf-8", + ) + overrides_file = tmp_path / "overrides.yaml" + overrides_file.write_text("overrides:\n cloudpickle: BSD-3-Clause\n", encoding="utf-8") + output_file = license_dir / "licenses.jsonl" + + format_licenses(osv_json, output_file, overrides_file, format_type="jsonl") + + assert output_file.read_text(encoding="utf-8") == ( + '{"name": "cloudpickle", "license": "BSD-3-CLAUSE", "compatible": true}' + )